Merge branch 'litellm_internal_staging' into deepkeep-as-internal

.github/workflows/test-linting.yml

@@ -14,11 +14,15 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 10
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        # Check out the PR head, not the default refs/pull/N/merge: the merge ref
+        # folds in newer base commits, which the diff-based gates (ruff delta,
+        # Any-discipline) would otherwise blame on this branch.
         with:
+          ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
           clean: true
           persist-credentials: false
@@ -73,6 +77,12 @@ jobs:
         run: |
           uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
 
+      - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
+
       - name: Print OpenAI version
         run: |
           uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
@@ -80,8 +90,11 @@ jobs:
       - name: Run MyPy type checking
         run: |
           cd litellm
-          uv run --no-sync mypy .
-          cd ..
+          (uv run --no-sync mypy . || true) | uv run --no-sync python ../scripts/type_check_gate.py --tool mypy
+
+      - name: Run basedpyright type checking
+        run: |
+          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --tool basedpyright
 
       - name: Check for circular imports
         run: |
@@ -93,6 +106,83 @@ jobs:
         run: |
           uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
+  # Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is
+  # raised (or a rule/budget is dropped) so a loosening is obvious in review, but it
+  # must be kept OUT of the branch-protection required-checks list so a justified
+  # bump can still be merged by a human who has seen and accepted the red.
+  budget-ratchet:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Ratchet check (budgets may only decrease; non-gating)
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          python scripts/budget_ratchet_check.py --base "$BASE_SHA"
+
+  any-discipline:
+    # Separate job: the first run cold-builds litellm's type cache (~2 min, ~3 GB),
+    # so keep it off the main lint job's time budget. Subsequent runs reuse the
+    # cached .mypy_cache_any and only re-type-check the changed files.
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        # Check out the PR head, not the default refs/pull/N/merge: the merge ref
+        # folds in newer base commits, which the diff-based gates (ruff delta,
+        # Any-discipline) would otherwise blame on this branch.
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+          clean: true
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          uv sync --frozen
+
+      # Keyed on deps + mypy config (which fix the type cache's validity), not on
+      # source content, so changed files always differ from the restored cache.
+      # The gate also defensively invalidates each target's cache entry, so
+      # correctness never depends on cache freshness -- this is purely for speed.
+      - name: Restore Any-gate type cache
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: .mypy_cache_any
+          key: any-mypy-cache-${{ runner.os }}-py3.12-${{ hashFiles('uv.lock', 'litellm/mypy.ini') }}
+          restore-keys: |
+            any-mypy-cache-${{ runner.os }}-py3.12-
+
+      - name: Check Any discipline on changed lines
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          uv run --no-sync python scripts/check_any_discipline.py --changed --base "$BASE_SHA"
+
   secret-scan:
     runs-on: ubuntu-latest
     timeout-minutes: 5

.gitignore

@@ -77,6 +77,7 @@ tests/local_testing/log.txt
 litellm/proxy/_new_new_secret_config.yaml
 litellm/proxy/custom_guardrail.py
 **/.mypy_cache/
+**/.mypy_cache_any/
 litellm/proxy/application.log
 tests/llm_translation/vertex_test_account.json
 tests/llm_translation/test_vertex_key.json

CLAUDE.md

@@ -36,7 +36,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
 
 Run tests, format your code, and lint your code before each commit
 
-When you fix strict-rule violations gated by `ruff-strict-budget.json`, run `make lint-strict-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom
+When you fix violations gated by `ruff-strict-budget.json`, `mypy-code-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered baselines so the ceilings ratchet down instead of leaving stale headroom
+
+If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and bringing it closer to the max, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
+
+The Any-discipline gate (`make lint-any`, also a CI job) fails when a line you changed under `litellm/` holds a value typed `Any`, including the `X | Any`. Ideally `# any-ok: <reason>` is never used; treat it as a last resort for a genuine typed/untyped boundary that Pydantic truly can't model
+
+If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
 Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it)
 
@@ -67,8 +73,6 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No file sprawl: deliberate file and folder structure
 - Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions
 
-if you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and bringing it closer to the max, just validate it in the caller (a simple function that returns the typed thing or raises will do) and then pass the now typed variable in
-
 Follow conventional commits for commit names and PR titles
 
 ## Think Before Coding

CONTRIBUTING.md

@@ -155,6 +155,7 @@ Individual linting commands:
 make format-check       # Check Black formatting
 make lint-ruff          # Run Ruff linting
 make lint-mypy          # Run MyPy type checking
+make lint-any           # Fail on Any-typed values on changed lines
 make check-circular-imports    # Check for circular imports
 make check-import-safety       # Check import safety
 ```

Makefile

@@ -5,7 +5,8 @@
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
 	info lint lint-dev format \
-	lint-strict-budget lint-strict-budget-update \
+	lint-mypy lint-mypy-budget-update lint-basedpyright lint-basedpyright-budget-update \
+	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-any \
 	install-dev install-proxy-dev install-test-deps install-hooks \
 	install-helm-unittest check-circular-imports check-import-safety
 
@@ -23,10 +24,15 @@ help:
 	@echo "  make format-check       - Check Black code formatting (matches CI)"
 	@echo "  make lint               - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
 	@echo "  make lint-ruff          - Run Ruff linting only"
-	@echo "  make lint-mypy          - Run MyPy type checking only"
+	@echo "  make lint-mypy          - Run MyPy (disallow_untyped_defs), gated by per-rule error counts"
+	@echo "  make lint-mypy-budget-update - Re-capture the MyPy per-rule budget (ratchet)"
+	@echo "  make lint-basedpyright  - Run basedpyright strict, gated by per-rule error counts"
+	@echo "  make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
 	@echo "  make lint-black         - Check Black formatting (matches CI)"
-	@echo "  make lint-strict-budget - Gate the codebase total of each strict ruff rule against its ceiling"
-	@echo "  make lint-strict-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
+	@echo "  make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
+	@echo "  make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
+	@echo "  make lint-budget-update - Re-capture all three ratchet budgets (ruff + mypy + basedpyright)"
+	@echo "  make lint-any           - Fail if changed lines under litellm/ hold an Any-typed value"
 	@echo "  make check-circular-imports - Check for circular imports"
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
@@ -121,7 +127,16 @@ lint-ruff-FULL-dev: install-dev
 	else echo "No changed .py files to check."; fi
 
 lint-mypy: install-dev
-	cd litellm && $(UV_RUN) mypy . --ignore-missing-imports && cd ..
+	cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy
+
+lint-mypy-budget-update: install-dev
+	cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy --update
+
+lint-basedpyright: install-dev
+	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright
+
+lint-basedpyright-budget-update: install-dev
+	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright --update
 
 lint-black: format-check
 
@@ -129,9 +144,18 @@ lint-strict-budget: install-dev
 	$(UV_RUN) python scripts/ruff_strict_gate.py \
 		--base $$(git rev-parse --verify origin/litellm_internal_staging 2>/dev/null && echo origin/litellm_internal_staging || echo upstream/litellm_internal_staging)
 
-lint-strict-budget-update: install-dev
+lint-ruff-budget: install-dev
+	$(UV_RUN) python scripts/ruff_strict_gate.py
+
+lint-ruff-budget-update: install-dev
 	$(UV_RUN) python scripts/ruff_strict_gate.py --update
 
+# Ratchet all three budgets in one shot (ruff strict + mypy + basedpyright)
+lint-budget-update: lint-ruff-budget-update lint-mypy-budget-update lint-basedpyright-budget-update
+
+lint-any: install-dev
+	$(UV_RUN) python scripts/check_any_discipline.py --changed
+
 check-circular-imports: install-dev
 	cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
@@ -139,10 +163,10 @@ check-import-safety: install-dev
 	@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
 # Combined linting (matches test-linting.yml workflow)
-lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety lint-strict-budget
+lint: format-check lint-ruff lint-mypy lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget lint-any
 
 # Faster linting for local development (only checks changed code)
-lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
+lint-dev: lint-format-changed lint-mypy lint-any check-circular-imports check-import-safety
 
 # Testing targets
 test: install-test-deps

README.md

@@ -327,6 +327,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 | [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |

basedpyright-code-budget.json

@@ -0,0 +1,194 @@
+{
+  "reportAny": {
+    "baseline": 24954,
+    "slack": 2500
+  },
+  "reportArgumentType": {
+    "baseline": 1863,
+    "slack": 3
+  },
+  "reportAssignmentType": {
+    "baseline": 220,
+    "slack": 3
+  },
+  "reportAttributeAccessIssue": {
+    "baseline": 335,
+    "slack": 3
+  },
+  "reportCallIssue": {
+    "baseline": 77,
+    "slack": 10
+  },
+  "reportConstantRedefinition": {
+    "baseline": 39,
+    "slack": 3
+  },
+  "reportDeprecated": {
+    "baseline": 217,
+    "slack": 10
+  },
+  "reportDuplicateImport": {
+    "baseline": 28,
+    "slack": 3
+  },
+  "reportExplicitAny": {
+    "baseline": 6931,
+    "slack": 700
+  },
+  "reportFunctionMemberAccess": {
+    "baseline": 7,
+    "slack": 3
+  },
+  "reportGeneralTypeIssues": {
+    "baseline": 151,
+    "slack": 3
+  },
+  "reportIncompatibleMethodOverride": {
+    "baseline": 52,
+    "slack": 10
+  },
+  "reportIncompatibleVariableOverride": {
+    "baseline": 8,
+    "slack": 3
+  },
+  "reportInconsistentOverload": {
+    "baseline": 12,
+    "slack": 3
+  },
+  "reportIndexIssue": {
+    "baseline": 26,
+    "slack": 3
+  },
+  "reportInvalidTypeForm": {
+    "baseline": 23,
+    "slack": 3
+  },
+  "reportInvalidTypeVarUse": {
+    "baseline": 2,
+    "slack": 3
+  },
+  "reportMatchNotExhaustive": {
+    "baseline": 1,
+    "slack": 3
+  },
+  "reportMissingParameterType": {
+    "baseline": 3933,
+    "slack": 10
+  },
+  "reportMissingTypeArgument": {
+    "baseline": 10612,
+    "slack": 1000
+  },
+  "reportMissingTypeStubs": {
+    "baseline": 27,
+    "slack": 10
+  },
+  "reportOperatorIssue": {
+    "baseline": 6,
+    "slack": 3
+  },
+  "reportOptionalCall": {
+    "baseline": 4,
+    "slack": 3
+  },
+  "reportOptionalIterable": {
+    "baseline": 3,
+    "slack": 3
+  },
+  "reportOptionalMemberAccess": {
+    "baseline": 724,
+    "slack": 10
+  },
+  "reportOptionalOperand": {
+    "baseline": 3,
+    "slack": 3
+  },
+  "reportOptionalSubscript": {
+    "baseline": 11,
+    "slack": 3
+  },
+  "reportPossiblyUnboundVariable": {
+    "baseline": 52,
+    "slack": 10
+  },
+  "reportPrivateUsage": {
+    "baseline": 1625,
+    "slack": 10
+  },
+  "reportRedeclaration": {
+    "baseline": 8,
+    "slack": 3
+  },
+  "reportReturnType": {
+    "baseline": 118,
+    "slack": 10
+  },
+  "reportTypedDictNotRequiredAccess": {
+    "baseline": 20,
+    "slack": 3
+  },
+  "reportUndefinedVariable": {
+    "baseline": 2,
+    "slack": 3
+  },
+  "reportUnknownArgumentType": {
+    "baseline": 30603,
+    "slack": 3000
+  },
+  "reportUnknownLambdaType": {
+    "baseline": 76,
+    "slack": 10
+  },
+  "reportUnknownMemberType": {
+    "baseline": 27322,
+    "slack": 2500
+  },
+  "reportUnknownParameterType": {
+    "baseline": 13636,
+    "slack": 1000
+  },
+  "reportUnknownVariableType": {
+    "baseline": 21776,
+    "slack": 2000
+  },
+  "reportUnnecessaryCast": {
+    "baseline": 118,
+    "slack": 10
+  },
+  "reportUnnecessaryComparison": {
+    "baseline": 680,
+    "slack": 10
+  },
+  "reportUnnecessaryContains": {
+    "baseline": 4,
+    "slack": 3
+  },
+  "reportUnnecessaryIsInstance": {
+    "baseline": 807,
+    "slack": 10
+  },
+  "reportUntypedBaseClass": {
+    "baseline": 110,
+    "slack": 3
+  },
+  "reportUntypedFunctionDecorator": {
+    "baseline": 22,
+    "slack": 3
+  },
+  "reportUnusedClass": {
+    "baseline": 22,
+    "slack": 3
+  },
+  "reportUnusedFunction": {
+    "baseline": 137,
+    "slack": 10
+  },
+  "reportUnusedImport": {
+    "baseline": 670,
+    "slack": 10
+  },
+  "reportUnusedVariable": {
+    "baseline": 865,
+    "slack": 10
+  }
+}

codecov.yaml

@@ -35,6 +35,22 @@ component_management:
     - component_id: "Enterprise"
       paths:
         - "enterprise/**"
+    - component_id: "Batches"
+      paths:
+        - "*/proxy/batches_endpoints/**"
+        - "litellm/batches/**"
+        - "*/llms/*/batches/**"
+    - component_id: "Videos"
+      paths:
+        - "litellm/videos/**"
+        - "*/proxy/video_endpoints/**"
+        - "*/llms/*/videos/**"
+    - component_id: "Realtime"
+      paths:
+        - "litellm/realtime_api/**"
+        - "*/proxy/realtime_endpoints/**"
+        - "*/llms/*/realtime/**"
+        - "litellm/litellm_core_utils/realtime_streaming.py"
 comment:
   layout: "header, diff, flags, components"  # show component info in the PR comment
 

db_scripts/create_views.py

@@ -15,7 +15,7 @@ db = Prisma(
 )
 
 
-async def check_view_exists():  # noqa: PLR0915
+async def check_view_exists():
     """
     Checks if the LiteLLM_VerificationTokenView and MonthlyGlobalSpend exists in the user's db.
 
@@ -34,8 +34,7 @@ async def check_view_exists():  # noqa: PLR0915
         print("LiteLLM_VerificationTokenView Exists!")  # noqa
     except Exception:
         # If an error occurs, the view does not exist, so create it
-        await db.execute_raw(
-            """
+        await db.execute_raw("""
                 CREATE VIEW "LiteLLM_VerificationTokenView" AS
                 SELECT 
                 v.*, 
@@ -45,8 +44,7 @@ async def check_view_exists():  # noqa: PLR0915
                 t.rpm_limit AS team_rpm_limit
                 FROM "LiteLLM_VerificationToken" v
                 LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id;
-            """
-        )
+            """)
 
         print("LiteLLM_VerificationTokenView Created!")  # noqa
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -412,7 +412,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
                     )
 
-    async def async_pre_call_hook(  # noqa: PLR0915
+    async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
         cache: DualCache,

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -483,7 +483,7 @@ async def new_project(
     response_model=LiteLLM_ProjectTable,
 )
 @management_endpoint_wrapper
-async def update_project(  # noqa: PLR0915
+async def update_project(
     data: UpdateProjectRequest,
     http_request: Request,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),

litellm/__init__.py

@@ -73,6 +73,7 @@ from litellm.constants import (
     replicate_models,
     clarifai_models,
     huggingface_models,
+    modelscope_models,
     empower_models,
     together_ai_models,
     baseten_models,
@@ -900,6 +901,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
             heroku_models.add(key)
         elif value.get("litellm_provider") == "dashscope":
             dashscope_models.add(key)
+        elif value.get("litellm_provider") == "modelscope":
+            modelscope_models.add(key)
         elif value.get("litellm_provider") == "moonshot":
             moonshot_models.add(key)
         elif value.get("litellm_provider") == "publicai":
@@ -1019,6 +1022,7 @@ model_list = list(
     | zai_models
     | fal_ai_models
     | deepseek_models
+    | modelscope_models
     | azure_ai_models
     | voyage_models
     | infinity_models
@@ -1152,6 +1156,7 @@ models_by_provider: dict = {
     "elevenlabs": elevenlabs_models,
     "heroku": heroku_models,
     "dashscope": dashscope_models,
+    "modelscope": modelscope_models,
     "moonshot": moonshot_models,
     "publicai": publicai_models,
     "v0": v0_models,
@@ -1975,6 +1980,9 @@ if TYPE_CHECKING:
     from .llms.dashscope.rerank.transformation import (
         DashScopeRerankConfig as DashScopeRerankConfig,
     )
+    from .llms.modelscope.chat.transformation import (
+        ModelScopeChatConfig as ModelScopeChatConfig,
+    )
     from .llms.moonshot.chat.transformation import (
         MoonshotChatConfig as MoonshotChatConfig,
     )

litellm/_lazy_imports_registry.py

@@ -306,6 +306,7 @@ LLM_CONFIG_NAMES = (
     "GigaChatConfig",
     "GigaChatEmbeddingConfig",
     "DashScopeChatConfig",
+    "ModelScopeChatConfig",
     "MoonshotChatConfig",
     "DockerModelRunnerChatConfig",
     "V0ChatConfig",
@@ -1161,6 +1162,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
         ".llms.dashscope.chat.transformation",
         "DashScopeChatConfig",
     ),
+    "ModelScopeChatConfig": (
+        ".llms.modelscope.chat.transformation",
+        "ModelScopeChatConfig",
+    ),
     "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"),
     "DockerModelRunnerChatConfig": (
         ".llms.docker_model_runner.chat.transformation",

litellm/_logging.py

@@ -419,7 +419,7 @@ def _enable_debugging():
 def print_verbose(print_statement):
     try:
         if set_verbose:
-            print(redact_secrets(str(print_statement)))  # noqa
+            print(redact_secrets(str(print_statement)))  # noqa: T201
     except Exception:
         pass
 

litellm/_redis.py

@@ -311,7 +311,7 @@ def get_redis_url_from_environment():
     return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
 
 
-def _get_redis_client_logic(**env_overrides):  # noqa: PLR0915
+def _get_redis_client_logic(**env_overrides):
     """
     Common functionality across sync + async redis client implementations
     """
@@ -567,7 +567,7 @@ def get_redis_client(**env_overrides):
     return redis.Redis(**redis_kwargs)
 
 
-def get_redis_async_client(  # noqa: PLR0915
+def get_redis_async_client(
     connection_pool: Optional[async_redis.BlockingConnectionPool] = None,
     **env_overrides,
 ) -> Union[async_redis.Redis, async_redis.RedisCluster]:

litellm/a2a_protocol/main.py

@@ -436,7 +436,7 @@ def _build_streaming_logging_obj(
     return logging_obj
 
 
-async def asend_message_streaming(  # noqa: PLR0915
+async def asend_message_streaming(
     a2a_client: Optional["A2AClientType"] = None,
     request: Optional["SendStreamingMessageRequest"] = None,
     api_base: Optional[str] = None,

litellm/batches/main.py

@@ -157,7 +157,7 @@ async def acreate_batch(
 
 
 @client
-def create_batch(  # noqa: PLR0915
+def create_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,

litellm/caching/caching.py

@@ -27,7 +27,7 @@ from litellm.types.utils import EmbeddingResponse, all_litellm_params
 from .azure_blob_cache import AzureBlobCache
 from .base_cache import BaseCache
 from .disk_cache import DiskCache
-from .dual_cache import DualCache  # noqa
+from .dual_cache import DualCache  # noqa: F401
 from .gcs_cache import GCSCache
 from .in_memory_cache import InMemoryCache
 from .qdrant_semantic_cache import QdrantSemanticCache
@@ -41,7 +41,7 @@ def print_verbose(print_statement):
     try:
         verbose_logger.debug(print_statement)
         if litellm.set_verbose:
-            print(print_statement)  # noqa
+            print(print_statement)  # noqa: T201
     except Exception:
         pass
 

litellm/caching/caching_handler.py

@@ -394,7 +394,7 @@ class LLMCachingHandler:
                 return cr["model"]
         return None
 
-    def _process_async_embedding_cached_response(  # noqa: PLR0915
+    def _process_async_embedding_cached_response(
         self,
         final_embedding_cached_response: Optional[EmbeddingResponse],
         cached_result: List[Optional[CachedEmbedding]],

litellm/caching/qdrant_semantic_cache.py

@@ -28,7 +28,7 @@ from .base_cache import BaseCache
 class QdrantSemanticCache(BaseCache):
     CACHE_KEY_FIELD_NAME = "litellm_cache_key"
 
-    def __init__(  # noqa: PLR0915
+    def __init__(
         self,
         qdrant_api_base=None,
         qdrant_api_key=None,

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -693,7 +693,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         original_response = model_call_details.get("original_response")
         return cls._recover_output_items_from_raw_sse(original_response)
 
-    def transform_response(  # noqa: PLR0915
+    def transform_response(
         self,
         model: str,
         raw_response: "BaseModel",
@@ -1211,7 +1211,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         return self.chunk_parser(json.loads(str_line))
 
     @staticmethod
-    def translate_responses_chunk_to_openai_stream(  # noqa: PLR0915
+    def translate_responses_chunk_to_openai_stream(
         parsed_chunk: Union[dict, BaseModel],
     ) -> "ModelResponseStream":
         """
@@ -1293,9 +1293,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                         provider_specific_fields
                     )
 
+                from litellm.responses.litellm_completion_transformation.transformation import (
+                    LiteLLMCompletionResponsesConfig,
+                )
+
                 tool_call_index = parsed_chunk.get("output_index", 0)
                 tool_call_chunk = ChatCompletionToolCallChunk(
-                    id=output_item.get("call_id"),
+                    id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(
+                        output_item.get("id"), output_item.get("call_id")
+                    ),
                     index=tool_call_index,
                     type="function",
                     function=function_chunk,

litellm/constants.py

@@ -190,6 +190,10 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
 # Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails)
 MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 100)
 
+# Metadata key recording which pre_call guardrails the proxy loop already ran,
+# so the deployment-level hook does not re-run them for the same request
+PRE_CALL_EXECUTED_GUARDRAILS_KEY = "_pre_call_executed_guardrails"
+
 # Generic fallback for unknown models
 DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
@@ -618,6 +622,7 @@ LITELLM_CHAT_PROVIDERS = [
     "nscale",
     "nebius",
     "dashscope",
+    "modelscope",
     "moonshot",
     "publicai",
     "v0",
@@ -776,6 +781,7 @@ openai_compatible_endpoints: List = [
     "inference.api.nscale.com/v1",
     "api.studio.nebius.ai/v1",
     "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
+    "https://api-inference.modelscope.cn/v1",
     "https://api.moonshot.ai/v1",
     "https://api.publicai.co/v1",
     "https://api.synthetic.new/openai/v1",
@@ -793,6 +799,7 @@ openai_compatible_endpoints: List = [
     "https://ai-gateway.vercel.sh/v1",
     "https://api.inference.wandb.ai/v1",
     "https://api.clarifai.com/v2/ext/openai/v1",
+    "https://api.libertai.io/v1",
 ]
 
 
@@ -836,10 +843,12 @@ openai_compatible_providers: List = [
     "poe",  # Poe - JSON-configured provider
     "chutes",  # Chutes - JSON-configured provider
     "parasail",  # Parasail - JSON-configured provider
+    "libertai",  # LibertAI - JSON-configured provider
     "featherless_ai",
     "nscale",
     "nebius",
     "dashscope",
+    "modelscope",
     "moonshot",
     "v0",
     "helicone",
@@ -865,6 +874,7 @@ openai_text_completion_compatible_providers: List = (
         "featherless_ai",
         "nebius",
         "dashscope",
+        "modelscope",
         "moonshot",
         "publicai",
         "synthetic",
@@ -1125,6 +1135,48 @@ WANDB_MODELS: set = set(
     ]
 )
 
+modelscope_models: set = set(
+    [
+        # Qwen series models
+        "Qwen/Qwen3-0.6B",
+        "Qwen/Qwen3-1.7B",
+        "Qwen/Qwen3-4B",
+        "Qwen/Qwen3-8B",
+        "Qwen/Qwen3-14B",
+        "Qwen/Qwen3-30B-A3B",
+        "Qwen/Qwen3-32B",
+        "Qwen/Qwen3-235B-A22B",
+        "Qwen/Qwen3-235B-A22B-Instruct-2507",
+        "Qwen/Qwen3-235B-A22B-Thinking-2507",
+        "Qwen/Qwen3-30B-A3B-Thinking-2507",
+        "Qwen/Qwen3-Coder-30B-A3B-Instruct",
+        "Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        "Qwen/Qwen3-Next-80B-A3B-Instruct",
+        "Qwen/Qwen3-Next-80B-A3B-Thinking",
+        "Qwen/Qwen3-VL-235B-A22B-Instruct",
+        "Qwen/Qwen3-VL-8B-Instruct",
+        "Qwen/Qwen3-VL-8B-Thinking",
+        "Qwen/Qwen3.5-122B-A10B",
+        "Qwen/Qwen3.5-27B",
+        "Qwen/Qwen3.5-35B-A3B",
+        "Qwen/Qwen3.5-397B-A17B",
+        "Qwen/QwQ-32B",
+        "Qwen/QwQ-32B-Preview",
+        "Qwen/QVQ-72B-Preview",
+        "Qwen/Qwen-Image-Edit",
+        # DeepSeek series models
+        "deepseek-ai/DeepSeek-R1-0528",
+        "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+        "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
+        "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
+        "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
+        "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
+        "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
+        "deepseek-ai/DeepSeek-V3.2",
+        "deepseek-ai/DeepSeek-V4-Flash",
+    ]
+)
+
 BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
     "cohere",
     "anthropic",

litellm/cost_calculator.py

@@ -288,7 +288,7 @@ def _transcription_usage_has_token_details(
     return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
 
 
-def cost_per_token(  # noqa: PLR0915
+def cost_per_token(
     model: str = "",
     prompt_tokens: int = 0,
     completion_tokens: int = 0,
@@ -1136,7 +1136,7 @@ def _store_cost_breakdown_in_logging_obj(
         pass
 
 
-def completion_cost(  # noqa: PLR0915
+def completion_cost(
     completion_response=None,
     model: Optional[str] = None,
     prompt="",

litellm/images/main.py

@@ -195,7 +195,7 @@ def image_generation(
 
 
 @client
-def image_generation(  # noqa: PLR0915
+def image_generation(
     prompt: str,
     model: Optional[str] = None,
     n: Optional[int] = None,
@@ -738,7 +738,7 @@ def image_variation(
 
 
 @client
-def image_edit(  # noqa: PLR0915
+def image_edit(
     image: Optional[Union[FileTypes, List[FileTypes]]] = None,
     prompt: Optional[str] = None,
     model: Optional[str] = None,

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -351,7 +351,7 @@ class SlackAlerting(CustomBatchLogger):
         except Exception:
             return 0
 
-    async def send_daily_reports(self, router) -> bool:  # noqa: PLR0915
+    async def send_daily_reports(self, router) -> bool:
         """
         Send a daily report on:
         - Top 5 deployments with most failed requests
@@ -1179,7 +1179,7 @@ Model Info:
         if response.status_code == 200:
             return True
         else:
-            print("Error sending webhook alert. Error=", response.text)  # noqa
+            print("Error sending webhook alert. Error=", response.text)  # noqa: T201
 
         return False
 
@@ -1373,7 +1373,7 @@ Model Info:
 
         return False
 
-    async def send_alert(  # noqa: PLR0915
+    async def send_alert(
         self,
         message: str,
         level: Literal["Low", "Medium", "High"],

litellm/integrations/anthropic_cache_control_hook.py

@@ -27,6 +27,11 @@ else:
     LiteLLMLoggingObj = Any
 
 
+# Anthropic (and Bedrock Claude) reject requests with more than 4 cache_control
+# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
+MAX_CACHE_CONTROL_BLOCKS = 4
+
+
 class AnthropicCacheControlHook(CustomPromptManagement):
     def get_chat_completion_prompt(
         self,
@@ -61,16 +66,30 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         processed_messages = copy.deepcopy(messages)
 
         # Separate message-level and non-message-level injection points
-        remaining_points = []
+        message_points: List[CacheControlMessageInjectionPoint] = []
+        remaining_points: List[CacheControlInjectionPoint] = []
         for point in injection_points:
             if point.get("location") == "message":
-                point = cast(CacheControlMessageInjectionPoint, point)
-                processed_messages = self._process_message_injection(
-                    point=point, messages=processed_messages
-                )
+                message_points.append(cast(CacheControlMessageInjectionPoint, point))
             else:
                 remaining_points.append(point)
 
+        # Non-message points (currently Bedrock tool_config) are handled in the
+        # provider transform, where each tool_config point appends at most one
+        # cachePoint to the tools. That block also counts toward Anthropic's
+        # limit, so reserve a slot for it here to leave room.
+        reserved_blocks = (
+            1
+            if any(p.get("location") == "tool_config" for p in remaining_points)
+            else 0
+        )
+
+        processed_messages = self._apply_message_injections(
+            points=message_points,
+            messages=processed_messages,
+            max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
+        )
+
         # Pass through non-message injection points for provider-specific handling
         if remaining_points:
             non_default_params["cache_control_injection_points"] = remaining_points
@@ -78,14 +97,71 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return model, processed_messages, non_default_params
 
     @staticmethod
-    def _process_message_injection(
-        point: CacheControlMessageInjectionPoint, messages: List[AllMessageValues]
+    def _apply_message_injections(
+        points: List[CacheControlMessageInjectionPoint],
+        messages: List[AllMessageValues],
+        max_blocks: int,
     ) -> List[AllMessageValues]:
-        """Process message-level cache control injection."""
-        control: ChatCompletionCachedContent = point.get(
-            "control", None
-        ) or ChatCompletionCachedContent(type="ephemeral")
+        """Apply message-level cache control injection points in order.
 
+        Anthropic allows at most ``MAX_CACHE_CONTROL_BLOCKS`` cache_control
+        breakpoints per request. Client-supplied breakpoints count toward that
+        limit, so we never inject onto a message that already carries
+        cache_control (preserving the client's TTL) and we stop injecting once
+        ``max_blocks`` is reached. Injection points are honored in config order,
+        so earlier points win when slots are scarce.
+        """
+        used_blocks = sum(
+            AnthropicCacheControlHook._count_cache_control_blocks(msg)
+            for msg in messages
+        )
+
+        limit_reached = False
+        for point in points:
+            if used_blocks >= max_blocks:
+                limit_reached = True
+                break
+
+            control: ChatCompletionCachedContent = point.get(
+                "control", None
+            ) or ChatCompletionCachedContent(type="ephemeral")
+
+            for target_index in AnthropicCacheControlHook._resolve_target_indices(
+                point=point, messages=messages
+            ):
+                if used_blocks >= max_blocks:
+                    limit_reached = True
+                    break
+
+                if AnthropicCacheControlHook._message_has_cache_control(
+                    messages[target_index]
+                ):
+                    # Client already marked this message; don't overwrite it.
+                    continue
+
+                messages[target_index] = (
+                    AnthropicCacheControlHook._safe_insert_cache_control_in_message(
+                        messages[target_index], control
+                    )
+                )
+                used_blocks += 1
+
+            if limit_reached:
+                break
+
+        if limit_reached:
+            verbose_logger.warning(
+                f"AnthropicCacheControlHook: Reached the Anthropic limit of "
+                f"{MAX_CACHE_CONTROL_BLOCKS} cache_control blocks. Skipping further injection."
+            )
+
+        return messages
+
+    @staticmethod
+    def _resolve_target_indices(
+        point: CacheControlMessageInjectionPoint, messages: List[AllMessageValues]
+    ) -> List[int]:
+        """Resolve which message indices an injection point targets."""
         _targetted_index: Optional[Union[int, str]] = point.get("index", None)
         targetted_index: Optional[int] = None
         if isinstance(_targetted_index, str):
@@ -96,36 +172,49 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         else:
             targetted_index = _targetted_index
 
-        targetted_role = point.get("role", None)
-
         # Case 1: Target by specific index
         if targetted_index is not None:
             original_index = targetted_index
-            # Handle negative indices (convert to positive)
             if targetted_index < 0:
                 targetted_index += len(messages)
 
             if 0 <= targetted_index < len(messages):
-                messages[targetted_index] = (
-                    AnthropicCacheControlHook._safe_insert_cache_control_in_message(
-                        messages[targetted_index], control
-                    )
-                )
-            else:
-                verbose_logger.warning(
-                    f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. "
-                    f"Targeted index was {targetted_index}. Skipping cache control injection for this point."
-                )
+                return [targetted_index]
+
+            verbose_logger.warning(
+                f"AnthropicCacheControlHook: Provided index {original_index} is out of bounds for message list of length {len(messages)}. "
+                f"Targeted index was {targetted_index}. Skipping cache control injection for this point."
+            )
+            return []
+
         # Case 2: Target by role
-        elif targetted_role is not None:
-            for msg in messages:
-                if msg.get("role") == targetted_role:
-                    msg = (
-                        AnthropicCacheControlHook._safe_insert_cache_control_in_message(
-                            message=msg, control=control
-                        )
-                    )
-        return messages
+        targetted_role = point.get("role", None)
+        if targetted_role is not None:
+            return [
+                idx
+                for idx, msg in enumerate(messages)
+                if msg.get("role") == targetted_role
+            ]
+
+        return []
+
+    @staticmethod
+    def _count_cache_control_blocks(message: AllMessageValues) -> int:
+        """Count cache_control breakpoints on a message (message + content level)."""
+        count = 0
+        if message.get("cache_control") is not None:
+            count += 1
+        content = message.get("content")
+        if isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("cache_control") is not None:
+                    count += 1
+        return count
+
+    @staticmethod
+    def _message_has_cache_control(message: AllMessageValues) -> bool:
+        """Return True if the message already carries any cache_control."""
+        return AnthropicCacheControlHook._count_cache_control_blocks(message) > 0
 
     @staticmethod
     def _safe_insert_cache_control_in_message(

litellm/integrations/braintrust_logging.py

@@ -133,9 +133,7 @@ class BraintrustLogger(CustomLogger):
 
         self.default_project_id = project_dict["id"]
 
-    def log_success_event(  # noqa: PLR0915
-        self, kwargs, response_obj, start_time, end_time
-    ):
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
         verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
         try:
             litellm_call_id = kwargs.get("litellm_call_id")
@@ -271,9 +269,7 @@ class BraintrustLogger(CustomLogger):
         except Exception as e:
             raise e  # don't use verbose_logger.exception, if exception is raised
 
-    async def async_log_success_event(  # noqa: PLR0915
-        self, kwargs, response_obj, start_time, end_time
-    ):
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
         try:
             litellm_call_id = kwargs.get("litellm_call_id")

litellm/integrations/custom_guardrail.py

@@ -1,3 +1,4 @@
+import secrets
 from datetime import datetime
 from typing import (
     TYPE_CHECKING,
@@ -43,6 +44,7 @@ if TYPE_CHECKING:
 dc = DualCache()
 
 
+from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY
 from litellm.exceptions import (
     BlockedPiiEntityError,
     GuardrailRaisedException,
@@ -50,6 +52,12 @@ from litellm.exceptions import (
     SensitiveDataRouteException,
 )
 
+# Per-process secret tagging each recorded marker. The deployment hook only
+# honors markers carrying this token, so a caller cannot forge the metadata
+# field to suppress a guardrail on the direct-SDK path that never reaches the
+# proxy's metadata sanitizer.
+_PRE_CALL_EXECUTED_TOKEN = secrets.token_hex(16)
+
 
 def get_session_id_from_request_data(request_data: Dict[str, Any]) -> Optional[str]:
     """Extract session_id from request data (litellm_session_id or metadata)."""
@@ -458,6 +466,49 @@ class CustomGuardrail(CustomLogger):
 
         return False
 
+    def _pre_call_marker(self) -> Optional[str]:
+        name = self.guardrail_name
+        if not name:
+            return None
+        return f"{_PRE_CALL_EXECUTED_TOKEN}:{name}"
+
+    def mark_pre_call_hook_ran(self, data: Dict[str, Any]) -> None:
+        """
+        Record that this guardrail's ``async_pre_call_hook`` already ran for this
+        request, so the deployment-level hook does not run it a second time.
+
+        The proxy runs pre-call guardrails in ``ProxyLogging.pre_call_hook``. The
+        router later spreads a deployment's model-level ``guardrails`` into the
+        top-level request kwargs, which would otherwise re-trigger the same hook
+        from ``async_pre_call_deployment_hook``.
+        """
+        marker = self._pre_call_marker()
+        if marker is None:
+            return
+        for meta_key in ("metadata", "litellm_metadata"):
+            meta = data.get(meta_key)
+            if isinstance(meta, dict):
+                executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
+                if isinstance(executed, list):
+                    if marker not in executed:
+                        executed.append(marker)
+                else:
+                    meta[PRE_CALL_EXECUTED_GUARDRAILS_KEY] = [marker]
+                return
+        data["metadata"] = {PRE_CALL_EXECUTED_GUARDRAILS_KEY: [marker]}
+
+    def _pre_call_hook_already_ran(self, data: Dict[str, Any]) -> bool:
+        marker = self._pre_call_marker()
+        if marker is None:
+            return False
+        for meta_key in ("metadata", "litellm_metadata"):
+            meta = data.get(meta_key)
+            if isinstance(meta, dict):
+                executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
+                if isinstance(executed, list) and marker in executed:
+                    return True
+        return False
+
     async def async_pre_call_deployment_hook(
         self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
     ) -> Optional[dict]:
@@ -468,6 +519,9 @@ class CustomGuardrail(CustomLogger):
         if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
             return kwargs
 
+        if self._pre_call_hook_already_ran(kwargs):
+            return kwargs
+
         if (
             self.should_run_guardrail(
                 data=kwargs, event_type=GuardrailEventHooks.pre_call
@@ -567,6 +621,9 @@ class CustomGuardrail(CustomLogger):
         ):
             return False
 
+        if self.default_on is True and disable_global_guardrail is True:
+            return False
+
         if self.default_on is True and disable_global_guardrail is not True:
             if self._event_hook_is_event_type(event_type):
                 if isinstance(self.event_hook, Mode):

litellm/integrations/langfuse/langfuse.py

@@ -549,7 +549,7 @@ class LangFuseLogger:
             )
         )
 
-    def _log_langfuse_v2(  # noqa: PLR0915
+    def _log_langfuse_v2(
         self,
         user_id: Optional[str],
         metadata: dict,

litellm/integrations/lunary.py

@@ -75,16 +75,16 @@ class LunaryLogger:
             version = importlib.metadata.version("lunary")  # type: ignore
             # if version < 0.1.43 then raise ImportError
             if packaging.version.Version(version) < packaging.version.Version("0.1.43"):  # type: ignore
-                print(  # noqa
+                print(  # noqa: T201
                     "Lunary version outdated. Required: >= 0.1.43. Upgrade via 'pip install lunary --upgrade'"
                 )
                 raise ImportError
 
             self.lunary_client = lunary
         except ImportError:
-            print(  # noqa
+            print(  # noqa: T201
                 "Lunary not installed. Please install it using 'pip install lunary'"
-            )  # noqa
+            )
             raise ImportError
 
     def log_event(

litellm/integrations/mock_client_factory.py

@@ -107,7 +107,7 @@ def _is_url_match(url, matchers: List[str]) -> bool:
         return False
 
 
-def create_mock_client_factory(config: MockClientConfig):  # noqa: PLR0915
+def create_mock_client_factory(config: MockClientConfig):
     """
     Factory function that creates mock client functions based on configuration.
 

litellm/integrations/opentelemetry.py

@@ -2198,9 +2198,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
         return kv_pairs
 
-    def set_attributes(  # noqa: PLR0915
-        self, span: Span, kwargs, response_obj: Optional[Any]
-    ):
+    def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]):
         try:
             if self.callback_name == "langtrace":
                 from litellm.integrations.langtrace import LangtraceAttributes

litellm/integrations/otel/README.md

@@ -216,7 +216,13 @@ lives in [`plumbing/`](./plumbing):
   `TracerProvider` so one logger serves many tenants. The cache is a bounded LRU
   that flushes + shuts down evicted providers, since the key derives from
   request-supplied credentials and must not grow (or leak threads) without limit.
-- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments.
+- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments. The
+  six `gen_ai.client.*` histograms are recorded through the meter resolved by
+  `providers.resolve_meter_provider`: an injected provider wins (tests/DI),
+  otherwise the operator's globally configured `MeterProvider` is reused so its
+  readers/exporters receive them alongside the server metrics, and one is built
+  and registered as the global only when none is set (mirroring how V2 owns trace
+  export).
 
 ### Adapter
 

litellm/integrations/otel/logger.py

@@ -10,6 +10,7 @@ from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Span, Tracer, get_current_span, use_span
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.otel.model.baggage import promoted_baggage
 from litellm.integrations.otel.model.config import OpenTelemetryV2Config
@@ -36,9 +37,15 @@ from litellm.integrations.otel.model.payloads import (
     SpanError,
     is_mcp_tool_call,
 )
+from litellm.integrations.otel.plumbing.metrics import (
+    GenAIMetricRecorder,
+    create_genai_metrics,
+)
 from litellm.integrations.otel.plumbing.providers import (
     build_tracer_provider,
+    get_meter,
     get_tracer,
+    resolve_meter_provider,
 )
 from litellm.integrations.otel.plumbing.routing import TenantTracerCache
 from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
@@ -95,7 +102,7 @@ class OpenTelemetryV2(CustomLogger):
         callback_name: str | None = None,
         tracer_provider: TracerProvider | None = None,
         logger_provider: Any | None = None,  # reserved for OTel logs
-        meter_provider: Any | None = None,  # reserved for metrics
+        meter_provider: Any | None = None,
         **kwargs: Any,
     ) -> None:
         super().__init__(**kwargs)
@@ -107,6 +114,8 @@ class OpenTelemetryV2(CustomLogger):
             else build_tracer_provider(self.config)
         )
         self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
+        self._metrics_recorder = self._init_metrics(meter_provider)
+        self._metric_filter_error_logged = False
         self._emitter = SpanEmitter(
             self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)
         )
@@ -116,6 +125,20 @@ class OpenTelemetryV2(CustomLogger):
         self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict()
         self._init_otel_logger_on_litellm_proxy()
 
+    def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None":
+        """Create the six GenAI histograms when metrics are enabled, else ``None``.
+
+        ``meter_provider`` is an explicit override (tests inject one); otherwise the
+        provider is resolved from the OTel global so the operator's configured
+        readers/exporters receive the metrics, building and registering one only
+        when no global provider is set.
+        """
+        if not self.config.enable_metrics:
+            return None
+        provider = resolve_meter_provider(self.config, meter_provider)
+        meter = get_meter(provider, LITELLM_TRACER_NAME)
+        return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name)
+
     # ====================================================================== #
     #  Proxy global registration
     # ====================================================================== #
@@ -208,6 +231,25 @@ class OpenTelemetryV2(CustomLogger):
         if self._emit_mcp_tool_call(kwargs, start_time, end_time):
             return
         self._close_llm_call(kwargs, start_time, end_time)
+        self._record_metrics(kwargs, response_obj, start_time, end_time)
+
+    def _record_metrics(self, kwargs, response_obj, start_time, end_time) -> None:
+        """Record the GenAI metrics for a successful LLM call. Best-effort: a
+        recording failure (e.g. a malformed payload) must never break the span
+        close or the request itself."""
+        if self._metrics_recorder is None:
+            return
+        try:
+            self._metrics_recorder.record(kwargs, response_obj, start_time, end_time)
+        except ValueError as exc:
+            if not self._metric_filter_error_logged:
+                verbose_logger.error(
+                    "OpenTelemetryV2: invalid otel.attributes metric filter, metrics disabled: %s",
+                    exc,
+                )
+                self._metric_filter_error_logged = True
+        except Exception as exc:
+            verbose_logger.debug("OpenTelemetryV2: metric recording failed: %s", exc)
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         if self._emit_mcp_tool_call(kwargs, start_time, end_time):

litellm/integrations/otel/mappers/genai.py

@@ -10,7 +10,12 @@ table: one lambda per mapping operation, applied against the typed span data.
 from typing import Callable
 
 from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
-from litellm.integrations.otel.mappers.utils import collect, drop_none
+from litellm.integrations.otel.mappers.utils import (
+    collect,
+    drop_none,
+    output_messages,
+    serialize_messages,
+)
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
@@ -47,6 +52,8 @@ class GenAIMapper:
             else None
         ),
         GenAI.REQUEST_SEED: lambda d: d.request_params.seed,
+        GenAI.INPUT_MESSAGES: lambda d: serialize_messages(d.messages_in),
+        GenAI.OUTPUT_MESSAGES: lambda d: serialize_messages(output_messages(d)),
         GenAI.RESPONSE_MODEL: lambda d: d.response_model,
         GenAI.RESPONSE_ID: lambda d: d.response_id,
         GenAI.RESPONSE_FINISH_REASONS: lambda d: (

litellm/integrations/otel/model/config.py

@@ -184,6 +184,20 @@ class OpenTelemetryV2Config(BaseSettings):
         ),
     )
 
+    @field_validator("capture_message_content", mode="before")
+    @classmethod
+    def _normalize_capture_message_content(cls, value: object) -> object:
+        """Fold the capture mode to its canonical lower_snake_case form.
+
+        V1 read this env var case-insensitively, so operators set the
+        UPPER_SNAKE_CASE form (e.g. ``SPAN_AND_EVENT``). The canonical values
+        here are lower_snake_case; normalizing at the boundary keeps both
+        spellings working and lets every downstream comparison stay exact.
+        """
+        if isinstance(value, str):
+            return value.lower()
+        return value
+
     @field_validator(
         "baggage_promoted_keys",
         "baggage_metadata_keys",

litellm/integrations/otel/model/semconv.py

@@ -230,6 +230,10 @@ class Metric:
 
     TOKEN_USAGE: Final = "gen_ai.client.token.usage"
     OPERATION_DURATION: Final = "gen_ai.client.operation.duration"
+    TOKEN_COST: Final = "gen_ai.client.token.cost"
+    TIME_TO_FIRST_TOKEN: Final = "gen_ai.client.response.time_to_first_token"
+    TIME_PER_OUTPUT_TOKEN: Final = "gen_ai.client.response.time_per_output_token"
+    RESPONSE_DURATION: Final = "gen_ai.client.response.duration"
 
 
 # litellm ``custom_llm_provider`` -> ``gen_ai.provider.name`` value.

litellm/integrations/otel/plumbing/metrics.py

@@ -1,28 +1,265 @@
-"""GenAI client metrics (token usage + operation duration histograms)."""
+"""GenAI client metrics: the six ``gen_ai.client.*`` histograms plus the
+recorder that builds attributes, applies the shared cardinality filter, and
+records a request's metrics in the success path.
+
+The instrument names/units/descriptions and the recording + timing math mirror
+the v1 :mod:`litellm.integrations.opentelemetry` integration so both engines emit
+identical metrics. The attribute cardinality filter is reused from v1 by import
+(no duplication of the valid-name set or its validation).
+"""
 
 from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, FrozenSet, Mapping, Optional
 
 from opentelemetry.metrics import Histogram, Meter
 
-from litellm.integrations.otel.model.semconv import Metric
+import litellm
+from litellm.integrations.opentelemetry import (
+    METRIC_METADATA_KEYS,
+    TOKEN_TYPE_ATTRIBUTE,
+    _build_metric_attribute_filter,
+    _resolve_metric_attribute_filter,
+)
+from litellm.integrations.otel.model.semconv import Metric, resolve_operation
+from litellm.integrations.otel.model.utils import to_seconds
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
 
 @dataclass(frozen=True)
 class GenAIMetrics:
-    token_usage: Histogram
     operation_duration: Histogram
+    token_usage: Histogram
+    token_cost: Histogram
+    time_to_first_token: Histogram
+    time_per_output_token: Histogram
+    response_duration: Histogram
 
 
 def create_genai_metrics(meter: Meter) -> GenAIMetrics:
     return GenAIMetrics(
-        token_usage=meter.create_histogram(
-            name=Metric.TOKEN_USAGE,
-            unit="{token}",
-            description="Number of tokens used per GenAI request.",
-        ),
         operation_duration=meter.create_histogram(
             name=Metric.OPERATION_DURATION,
             unit="s",
-            description="GenAI operation duration.",
+            description="GenAI operation duration",
+        ),
+        token_usage=meter.create_histogram(
+            name=Metric.TOKEN_USAGE,
+            unit="{token}",
+            description="GenAI token usage",
+        ),
+        token_cost=meter.create_histogram(
+            name=Metric.TOKEN_COST,
+            unit="USD",
+            description="GenAI request cost",
+        ),
+        time_to_first_token=meter.create_histogram(
+            name=Metric.TIME_TO_FIRST_TOKEN,
+            unit="s",
+            description="Time to first token for streaming requests",
+        ),
+        time_per_output_token=meter.create_histogram(
+            name=Metric.TIME_PER_OUTPUT_TOKEN,
+            unit="s",
+            description="Average time per output token (generation time / completion tokens)",
+        ),
+        response_duration=meter.create_histogram(
+            name=Metric.RESPONSE_DURATION,
+            unit="s",
+            description="Total LLM API generation time (excludes LiteLLM overhead)",
         ),
     )
+
+
+class GenAIMetricRecorder:
+    """Records the six GenAI histograms for one successful LLM call.
+
+    The cardinality filter is resolved lazily on the first record: the proxy
+    populates ``callback_settings.otel.attributes`` after the logger is built, so
+    reading it at construction time would miss it. ``gen_ai.token.type`` is added
+    to the token-usage attributes after filtering so the input/output split always
+    survives.
+    """
+
+    def __init__(
+        self, metrics: GenAIMetrics, callback_name: Optional[str] = None
+    ) -> None:
+        self._metrics = metrics
+        self._callback_name = callback_name
+        self._include: Optional[FrozenSet[str]] = None
+        self._exclude: Optional[FrozenSet[str]] = None
+        self._filter_resolved = False
+
+    def record(
+        self,
+        kwargs: Mapping[str, Any],
+        response_obj: Any,
+        start_time: datetime,
+        end_time: datetime,
+    ) -> None:
+        common_attrs = self._filter_attributes(self._common_attributes(kwargs))
+        duration_s = (end_time - start_time).total_seconds()
+
+        self._metrics.operation_duration.record(duration_s, attributes=common_attrs)
+        self._record_token_usage(response_obj, common_attrs)
+
+        cost = kwargs.get("response_cost")
+        if cost:
+            self._metrics.token_cost.record(cost, attributes=common_attrs)
+
+        self._record_time_to_first_token(kwargs, common_attrs)
+        self._record_time_per_output_token(
+            kwargs, response_obj, end_time, duration_s, common_attrs
+        )
+        self._record_response_duration(kwargs, end_time, common_attrs)
+
+    # ------------------------------------------------------------------ #
+    #  Attribute building + cardinality filter
+    # ------------------------------------------------------------------ #
+
+    def _common_attributes(self, kwargs: Mapping[str, Any]) -> dict:
+        params = kwargs.get("litellm_params") or {}
+        provider = params.get("custom_llm_provider", "Unknown")
+        common_attrs: dict = {
+            "gen_ai.operation.name": resolve_operation(kwargs.get("call_type")).value,
+            "gen_ai.system": provider,
+            "gen_ai.request.model": kwargs.get("model"),
+            "gen_ai.framework": "litellm",
+        }
+
+        std_log = kwargs.get("standard_logging_object")
+        md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
+        for key in METRIC_METADATA_KEYS:
+            value = md.get(key)
+            if value is None:
+                continue
+            if isinstance(value, (dict, list)):
+                common_attrs[f"metadata.{key}"] = safe_dumps(value)
+            else:
+                common_attrs[f"metadata.{key}"] = str(value)
+
+        hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
+            "hidden_params", {}
+        )
+        if hidden_params:
+            common_attrs["hidden_params"] = safe_dumps(hidden_params)
+
+        return common_attrs
+
+    def _ensure_filter(self) -> None:
+        if self._filter_resolved:
+            return
+        attributes = None
+        if self._callback_name in (None, "otel"):
+            otel_settings = (litellm.callback_settings or {}).get("otel") or {}
+            raw = (
+                otel_settings.get("attributes")
+                if isinstance(otel_settings, dict)
+                else None
+            )
+            if raw is not None:
+                attributes = _build_metric_attribute_filter(raw)
+        # A bad filter (include_list + exclude_list both set, an unfilterable name)
+        # raises here; the caller (logger._record_metrics) surfaces it once at ERROR
+        # so the operator-fixable config error is visible. Not cached on the raise
+        # path -- _filter_resolved stays False -- so a corrected config takes effect
+        # without reconstructing the recorder.
+        self._include, self._exclude = _resolve_metric_attribute_filter(attributes)
+        self._filter_resolved = True
+
+    def _filter_attributes(self, attrs: dict) -> dict:
+        self._ensure_filter()
+        if self._include is not None:
+            return {k: v for k, v in attrs.items() if k in self._include}
+        if self._exclude is not None:
+            return {k: v for k, v in attrs.items() if k not in self._exclude}
+        return attrs
+
+    # ------------------------------------------------------------------ #
+    #  Per-metric recording
+    # ------------------------------------------------------------------ #
+
+    def _record_token_usage(self, response_obj: Any, common_attrs: dict) -> None:
+        if not response_obj:
+            return
+        usage = response_obj.get("usage")
+        if not usage:
+            return
+        in_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
+        out_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"}
+        self._metrics.token_usage.record(
+            usage.get("prompt_tokens", 0), attributes=in_attrs
+        )
+        self._metrics.token_usage.record(
+            usage.get("completion_tokens", 0), attributes=out_attrs
+        )
+
+    def _record_time_to_first_token(
+        self, kwargs: Mapping[str, Any], common_attrs: dict
+    ) -> None:
+        if not kwargs.get("optional_params", {}).get("stream", False):
+            return
+        api_call_start = to_seconds(kwargs.get("api_call_start_time"))
+        completion_start = to_seconds(kwargs.get("completion_start_time"))
+        if api_call_start is None or completion_start is None:
+            return
+        self._metrics.time_to_first_token.record(
+            completion_start - api_call_start, attributes=common_attrs
+        )
+
+    def _record_time_per_output_token(
+        self,
+        kwargs: Mapping[str, Any],
+        response_obj: Any,
+        end_time: datetime,
+        duration_s: float,
+        common_attrs: dict,
+    ) -> None:
+        completion_tokens = None
+        if response_obj and (usage := response_obj.get("usage")):
+            completion_tokens = usage.get("completion_tokens")
+        if completion_tokens is None or completion_tokens <= 0:
+            return
+
+        end_ts = to_seconds(end_time)
+        if end_ts is None:
+            generation_time = duration_s
+        else:
+            completion_start_time = kwargs.get("completion_start_time")
+            api_call_start_time = kwargs.get("api_call_start_time")
+            if completion_start_time is not None:
+                completion_start = to_seconds(completion_start_time)
+                generation_time = (
+                    duration_s
+                    if completion_start is None
+                    else end_ts - completion_start
+                )
+            elif api_call_start_time is not None:
+                api_call_start = to_seconds(api_call_start_time)
+                generation_time = (
+                    duration_s if api_call_start is None else end_ts - api_call_start
+                )
+            else:
+                generation_time = duration_s
+
+        if generation_time > 0:
+            self._metrics.time_per_output_token.record(
+                generation_time / completion_tokens, attributes=common_attrs
+            )
+
+    def _record_response_duration(
+        self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict
+    ) -> None:
+        api_call_start_time = kwargs.get("api_call_start_time")
+        if api_call_start_time is None:
+            return
+        _end_time = kwargs.get("end_time") or end_time
+        if _end_time is None:
+            _end_time = datetime.now()
+        api_call_start = to_seconds(api_call_start_time)
+        end_ts = to_seconds(_end_time)
+        if api_call_start is None or end_ts is None:
+            return
+        duration = end_ts - api_call_start
+        if duration > 0:
+            self._metrics.response_duration.record(duration, attributes=common_attrs)

litellm/integrations/otel/plumbing/providers.py

@@ -1,9 +1,11 @@
 """Provider / exporter factory + the Baggage span processor."""
 
-from typing import Callable, Iterable
+from typing import TYPE_CHECKING, Any, Callable, Iterable
 
-from opentelemetry import baggage
+from opentelemetry import baggage, metrics
 from opentelemetry.context import Context
+from opentelemetry.metrics import MeterProvider, NoOpMeterProvider
+from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
 from opentelemetry.sdk.trace.export import (
@@ -25,6 +27,10 @@ from litellm.integrations.otel.model.spans import LiteLLMSpanKind
 # Re-exported so ``providers.parse_headers`` remains a stable entry point.
 from litellm.integrations.otel.model.utils import parse_headers as parse_headers
 
+if TYPE_CHECKING:
+    from opentelemetry.metrics import Meter
+    from opentelemetry.sdk.metrics.export import MetricReader
+
 _SPAN_KIND_BY_ROLE_KIND: dict[LiteLLMSpanKind, SpanKind] = {
     LiteLLMSpanKind.SERVER: SpanKind.SERVER,
     LiteLLMSpanKind.CLIENT: SpanKind.CLIENT,
@@ -157,6 +163,120 @@ def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
     )
 
 
+def _otlp_metrics_endpoint(endpoint: str | None) -> str | None:
+    """Point an OTLP/HTTP base endpoint at the ``/v1/metrics`` signal path.
+
+    The OTLP/HTTP exporter only appends ``/v1/metrics`` when it reads
+    ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used
+    verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint``
+    for the metrics signal (rewriting a sibling signal path when present).
+    """
+    if not endpoint:
+        return endpoint
+    endpoint = endpoint.rstrip("/")
+    if endpoint.endswith("/v1/metrics"):
+        return endpoint
+    for other_signal in ("/v1/traces", "/v1/logs"):
+        if endpoint.endswith(other_signal):
+            return endpoint[: -len(other_signal)] + "/v1/metrics"
+    return endpoint + "/v1/metrics"
+
+
+def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
+    """Build a metric reader mirroring v1's exporter selection.
+
+    ``console`` (and any unrecognized kind) exports to the console; ``otlp_http``
+    and ``otlp_grpc`` export over OTLP with the configured endpoint/headers. The
+    reader exports on a 5s period, matching v1.
+    """
+    from opentelemetry.sdk.metrics.export import (
+        ConsoleMetricExporter,
+        PeriodicExportingMetricReader,
+    )
+
+    kind = (config.exporter or "console").lower()
+    if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
+        from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
+            OTLPMetricExporter as HTTPMetricExporter,
+        )
+        from opentelemetry.sdk.metrics import Histogram
+        from opentelemetry.sdk.metrics.export import AggregationTemporality
+
+        exporter: Any = HTTPMetricExporter(
+            endpoint=_otlp_metrics_endpoint(config.endpoint),
+            headers=parse_headers(config.headers),
+            preferred_temporality={Histogram: AggregationTemporality.DELTA},
+        )
+    elif kind in ("otlp_grpc", "grpc"):
+        from opentelemetry.sdk.metrics import Histogram
+        from opentelemetry.sdk.metrics.export import AggregationTemporality
+
+        try:
+            from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+                OTLPMetricExporter as GRPCMetricExporter,
+            )
+        except ImportError as exc:
+            raise ImportError(
+                "OpenTelemetry OTLP gRPC metric exporter is not available. Install "
+                "`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)."
+            ) from exc
+
+        exporter = GRPCMetricExporter(
+            endpoint=config.endpoint,
+            headers=parse_headers(config.headers),
+            preferred_temporality={Histogram: AggregationTemporality.DELTA},
+        )
+    else:
+        exporter = ConsoleMetricExporter()
+
+    return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
+
+
+def build_meter_provider(
+    config: OpenTelemetryV2Config,
+    metric_reader: "MetricReader | None" = None,
+) -> SDKMeterProvider:
+    """Build the :class:`MeterProvider` for GenAI metrics.
+
+    ``metric_reader`` is an explicit override (tests inject an
+    ``InMemoryMetricReader``); otherwise the reader is selected from the config's
+    exporter kind via :func:`build_metric_reader`.
+    """
+    reader = metric_reader if metric_reader is not None else build_metric_reader(config)
+    return SDKMeterProvider(metric_readers=[reader], resource=build_resource(config))
+
+
+def resolve_meter_provider(
+    config: OpenTelemetryV2Config,
+    meter_provider: MeterProvider | None = None,
+) -> MeterProvider:
+    """Resolve the :class:`MeterProvider` GenAI metrics record through.
+
+    An injected provider wins (DI/tests). Otherwise reuse whatever the operator has
+    configured as the global, whether a real SDK provider or an explicit
+    ``NoOpMeterProvider``, so the GenAI histograms ride the operator's
+    readers/exporters and an explicit opt-out is honored. Only when the global is
+    still the default proxy placeholder does V2 build one from the config and
+    publish it as the global, mirroring how V2 owns trace export. The built
+    provider is the one returned, so its reader thread is always live, never
+    orphaned.
+    """
+    if meter_provider is not None:
+        return meter_provider
+
+    existing = metrics.get_meter_provider()
+    if isinstance(existing, (SDKMeterProvider, NoOpMeterProvider)):
+        return existing
+
+    provider = build_meter_provider(config)
+    metrics.set_meter_provider(provider)
+    return provider
+
+
+def get_meter(provider: MeterProvider, name: str = "litellm") -> "Meter":
+    return provider.get_meter(name, litellm_version)
+
+
 def build_resource(config: OpenTelemetryV2Config) -> Resource:
     attributes: dict[str, str] = {"service.name": config.service_name}
     if config.deployment_environment:

litellm/integrations/prometheus.py

@@ -75,7 +75,7 @@ class PrometheusLogger(CustomLogger):
                 return cb
         return None
 
-    def __init__(  # noqa: PLR0915
+    def __init__(
         self,
         **kwargs,
     ):
@@ -2255,7 +2255,7 @@ class PrometheusLogger(CustomLogger):
             or _litellm_params_metadata.get("user_agent"),
         }
 
-    def set_llm_deployment_failure_metrics(self, request_kwargs: dict):  # noqa: PLR0915
+    def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
         """
         Sets Failure metrics when an LLM API call fails
 

litellm/integrations/websearch_interception/handler.py

@@ -1128,7 +1128,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             )
             raise
 
-    async def _execute_chat_completion_agentic_loop(  # noqa: PLR0915
+    async def _execute_chat_completion_agentic_loop(
         self,
         model: str,
         messages: List[Dict],
@@ -1159,7 +1159,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             **request_patch.kwargs,
         )
 
-    async def _build_chat_completion_request_patch(  # noqa: PLR0915
+    async def _build_chat_completion_request_patch(
         self,
         model: str,
         messages: List[Dict],

litellm/integrations/weights_biases.py

@@ -21,10 +21,11 @@ try:
         # contains a (known) object attribute
         object: Literal["chat.completion", "edit", "text_completion"]
 
-        def __getitem__(self, key: K) -> V: ...  # noqa
+        def __getitem__(self, key: K) -> V: ...
 
-        def get(self, key: K, default: Optional[V] = None) -> Optional[V]:  # noqa
-            ...  # pragma: no cover
+        def get(
+            self, key: K, default: Optional[V] = None
+        ) -> Optional[V]: ...  # pragma: no cover
 
     class OpenAIRequestResponseResolver:
         def __call__(

litellm/litellm_core_utils/core_helpers.py

@@ -242,9 +242,28 @@ def _get_parent_otel_span_from_kwargs(
         return None
 
 
-def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> dict:
+def process_response_headers(
+    response_headers: Union[httpx.Headers, dict],
+    preserve_litellm_internal_headers: bool = False,
+) -> dict:
+    """
+    `preserve_litellm_internal_headers` must only be True when the input is a
+    LiteLLM-owned dict (e.g. `_hidden_params["additional_headers"]` that has
+    already been through one round of processing).  For raw upstream provider
+    headers — whether passed as `httpx.Headers` or a plain dict — it must
+    remain False, otherwise a malicious provider returning `x-litellm-*` could
+    spoof LiteLLM-internal markers (e.g. `x-litellm-attempted-fallbacks`).
+
+    When the input is an `httpx.Headers` object the flag is always treated as
+    False regardless of what the caller requested, because `httpx.Headers` is
+    always a raw provider response and can never be LiteLLM-owned.
+    """
     from litellm.types.utils import OPENAI_RESPONSE_HEADERS
 
+    # Raw httpx.Headers objects come directly from provider HTTP responses and
+    # must never be treated as LiteLLM-owned, regardless of caller intent.
+    _preserve = preserve_litellm_internal_headers and isinstance(response_headers, dict)
+
     openai_headers = {}
     processed_headers = {}
     additional_headers = {}
@@ -256,6 +275,12 @@ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> di
             "llm_provider-"
         ):  # return raw provider headers (incl. openai-compatible ones)
             processed_headers[k] = v
+        elif _preserve and k.startswith("x-litellm-"):
+            # LiteLLM's own internal headers (e.g. x-litellm-attempted-fallbacks,
+            # x-litellm-model-group) are not LLM provider headers and must not be
+            # prefixed. Downstream consumers (proxy override, callers checking
+            # whether a fallback happened) look up the bare key.
+            processed_headers[k] = v
         else:
             additional_headers["{}-{}".format("llm_provider", k)] = v
 

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -234,7 +234,7 @@ def extract_and_raise_litellm_exception(
                 )
 
 
-def exception_type(  # type: ignore  # noqa: PLR0915
+def exception_type(  # type: ignore
     model,
     original_exception,
     custom_llm_provider,
@@ -250,14 +250,14 @@ def exception_type(  # type: ignore  # noqa: PLR0915
     exception_mapping_worked = False
     exception_provider = custom_llm_provider
     if litellm.suppress_debug_info is False:
-        print()  # noqa
-        print(  # noqa
-            "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"  # noqa
-        )  # noqa
-        print(  # noqa
-            "LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'."  # noqa
-        )  # noqa
-        print()  # noqa
+        print()  # noqa: T201
+        print(  # noqa: T201
+            "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"
+        )
+        print(  # noqa: T201
+            "LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'."
+        )
+        print()  # noqa: T201
 
     litellm_response_headers = _get_response_headers(
         original_exception=original_exception

litellm/litellm_core_utils/fallback_utils.py

@@ -7,6 +7,9 @@ from litellm.litellm_core_utils.core_helpers import (
     safe_deep_copy,
     filter_internal_params,
 )
+from litellm.router_utils.add_retry_fallback_headers import (
+    add_fallback_headers_to_response,
+)
 
 from .asyncify import run_async_function
 
@@ -42,7 +45,7 @@ async def async_completion_with_fallbacks(**kwargs):
 
     # Try each fallback model
     most_recent_exception_str: Optional[str] = None
-    for fallback in fallbacks:
+    for attempted_fallbacks, fallback in enumerate(fallbacks):
         try:
             completion_kwargs = safe_deep_copy(base_kwargs)
             # Handle dictionary fallback configurations
@@ -63,7 +66,10 @@ async def async_completion_with_fallbacks(**kwargs):
             )
 
             if response is not None:
-                return response
+                return add_fallback_headers_to_response(
+                    response=response,
+                    attempted_fallbacks=attempted_fallbacks,
+                )
 
         except Exception as e:
             verbose_logger.exception(

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -154,7 +154,7 @@ def handle_anthropic_text_model_custom_llm_provider(
     return model, custom_llm_provider
 
 
-def get_llm_provider(  # noqa: PLR0915
+def get_llm_provider(
     model: str,
     custom_llm_provider: Optional[str] = None,
     api_base: Optional[str] = None,
@@ -334,6 +334,9 @@ def get_llm_provider(  # noqa: PLR0915
                     elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1":
                         custom_llm_provider = "dashscope"
                         dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY")
+                    elif endpoint == "https://api-inference.modelscope.cn/v1":
+                        custom_llm_provider = "modelscope"
+                        dynamic_api_key = get_secret_str("MODELSCOPE_API_KEY")
                     elif endpoint == "api.moonshot.ai/v1":
                         custom_llm_provider = "moonshot"
                         dynamic_api_key = get_secret_str("MOONSHOT_API_KEY")
@@ -526,11 +529,11 @@ def get_llm_provider(  # noqa: PLR0915
             custom_llm_provider = "sap"
         if not custom_llm_provider:
             if litellm.suppress_debug_info is False:
-                print()  # noqa
-                print(  # noqa
-                    "\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m"  # noqa
-                )  # noqa
-                print()  # noqa
+                print()  # noqa: T201
+                print(  # noqa: T201
+                    "\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m"
+                )
+                print()  # noqa: T201
             error_str = f"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={model}\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers"
             # maps to openai.NotFoundError, this is raised when openai does not recognize the llm
             raise litellm.exceptions.BadRequestError(  # type: ignore
@@ -565,7 +568,7 @@ def get_llm_provider(  # noqa: PLR0915
             )
 
 
-def _get_openai_compatible_provider_info(  # noqa: PLR0915
+def _get_openai_compatible_provider_info(
     model: str,
     api_base: Optional[str],
     api_key: Optional[str],
@@ -927,6 +930,13 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "modelscope":
+        (
+            api_base,
+            dynamic_api_key,
+        ) = litellm.ModelScopeChatConfig()._get_openai_compatible_provider_info(
+            api_base, api_key
+        )
     elif custom_llm_provider == "moonshot":
         (
             api_base,

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -5,7 +5,7 @@ from litellm.exceptions import BadRequestError
 from litellm.types.utils import LlmProviders, LlmProvidersSet
 
 
-def get_supported_openai_params(  # noqa: PLR0915
+def get_supported_openai_params(
     model: str,
     custom_llm_provider: Optional[str] = None,
     request_type: Literal[

litellm/litellm_core_utils/litellm_logging.py

@@ -986,7 +986,7 @@ class Logging(LiteLLMLoggingBaseClass):
             self._get_masked_api_base(additional_args.get("api_base", ""))
         )
 
-    def pre_call(self, input, api_key, model=None, additional_args={}):  # noqa: PLR0915
+    def pre_call(self, input, api_key, model=None, additional_args={}):
         # Log the exact input to the LLM API
         litellm.error_logs["PRE_CALL"] = locals()
         try:
@@ -2119,7 +2119,7 @@ class Logging(LiteLLMLoggingBaseClass):
             await self.async_success_handler(result=complete_streaming_response)
         return
 
-    def success_handler(  # noqa: PLR0915
+    def success_handler(
         self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs
     ):
         verbose_logger.debug(
@@ -2584,7 +2584,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 ),
             )
 
-    async def async_success_handler(  # noqa: PLR0915
+    async def async_success_handler(
         self, result=None, start_time=None, end_time=None, cache_hit=None, **kwargs
     ):
         """
@@ -3036,7 +3036,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     kwargs=self.model_call_details,
                 )  # type: ignore
 
-    def failure_handler(  # noqa: PLR0915
+    def failure_handler(
         self, exception, traceback_exception, start_time=None, end_time=None
     ):
         verbose_logger.debug(
@@ -3753,7 +3753,7 @@ def _get_masked_values(
     }
 
 
-def set_callbacks(callback_list, function_id=None):  # noqa: PLR0915
+def set_callbacks(callback_list, function_id=None):
     """
     Globally sets the callback client
     """
@@ -3854,7 +3854,7 @@ def set_callbacks(callback_list, function_id=None):  # noqa: PLR0915
     return None
 
 
-def _init_custom_logger_compatible_class(  # noqa: PLR0915
+def _init_custom_logger_compatible_class(
     logging_integration: _custom_logger_compatible_callbacks_literal,
     internal_usage_cache: Optional[DualCache],
     llm_router: Optional[
@@ -4611,7 +4611,7 @@ def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None:
         )
 
 
-def get_custom_logger_compatible_class(  # noqa: PLR0915
+def get_custom_logger_compatible_class(
     logging_integration: _custom_logger_compatible_callbacks_literal,
 ) -> Optional[CustomLogger]:
     try:
@@ -5893,7 +5893,7 @@ def get_standard_logging_object_payload(
 
 def emit_standard_logging_payload(payload: StandardLoggingPayload):
     if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
-        print(json.dumps(payload, indent=4))  # noqa
+        print(json.dumps(payload, indent=4))  # noqa: T201
 
 
 def get_standard_logging_metadata(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -683,7 +683,7 @@ def _get_regional_uplift_multiplier(
         return 1.0
 
 
-def generic_cost_per_token(  # noqa: PLR0915
+def generic_cost_per_token(
     model: str,
     usage: Usage,
     custom_llm_provider: str,

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -471,7 +471,7 @@ def _should_convert_tool_call_to_json_mode(
     return False
 
 
-def convert_to_model_response_object(  # noqa: PLR0915
+def convert_to_model_response_object(
     response_object: Optional[dict] = None,
     model_response_object: Optional[
         Union[

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -49,7 +49,8 @@ class ResponseMetadata:
                 result=self.result, litellm_model_name=model, router_model_id=model_id
             ),
             "additional_headers": process_response_headers(
-                self._get_value_from_hidden_params("additional_headers") or {}
+                self._get_value_from_hidden_params("additional_headers") or {},
+                preserve_litellm_internal_headers=True,
             ),
             "litellm_model_name": model,
         }

litellm/litellm_core_utils/logging_callback_manager.py

@@ -394,6 +394,22 @@ class LoggingCallbackManager:
             + litellm._async_failure_callback
         )
 
+    def remove_callback_from_all_lists(self, obj, require_self=False) -> None:
+        """
+        Remove a callback object from every callback list it may have been
+        promoted into, so a re-initialized callback leaves no stale instance behind.
+        """
+        for callback_list in (
+            litellm.callbacks,
+            litellm.success_callback,
+            litellm.failure_callback,
+            litellm._async_success_callback,
+            litellm._async_failure_callback,
+        ):
+            self.remove_callback_from_list_by_object(
+                callback_list, obj, require_self=require_self
+            )
+
     def get_active_additional_logging_utils_from_custom_logger(
         self,
     ) -> Set[AdditionalLoggingUtils]:

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1475,7 +1475,7 @@ def convert_to_gemini_tool_call_invoke(
         )
 
 
-def convert_to_gemini_tool_call_result(  # noqa: PLR0915
+def convert_to_gemini_tool_call_result(
     message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
     last_message_with_tool_calls: Optional[dict],
     model: Optional[str] = None,
@@ -2227,7 +2227,7 @@ def _sanitize_empty_text_content(
     return message
 
 
-def _add_missing_tool_results(  # noqa: PLR0915
+def _add_missing_tool_results(
     current_message: AllMessageValues,
     messages: List[AllMessageValues],
     current_index: int,
@@ -2484,7 +2484,7 @@ def sanitize_messages_for_tool_calling(
     return sanitized_messages
 
 
-def anthropic_messages_pt(  # noqa: PLR0915
+def anthropic_messages_pt(
     messages: List[AllMessageValues],
     model: str,
     llm_provider: str,
@@ -3278,7 +3278,7 @@ def convert_to_cohere_tool_invoke(tool_calls: list) -> List[ToolCallObject]:
     return cohere_tool_invoke
 
 
-def cohere_messages_pt_v2(  # noqa: PLR0915
+def cohere_messages_pt_v2(
     messages: List,
     model: str,
     llm_provider: str,
@@ -4703,7 +4703,7 @@ class BedrockConverseMessagesProcessor:
         return messages
 
     @staticmethod
-    async def _bedrock_converse_messages_pt_async(  # noqa: PLR0915
+    async def _bedrock_converse_messages_pt_async(
         messages: List,
         model: str,
         llm_provider: str,
@@ -5133,7 +5133,7 @@ class BedrockConverseMessagesProcessor:
         return assistant_parts
 
 
-def _bedrock_converse_messages_pt(  # noqa: PLR0915
+def _bedrock_converse_messages_pt(
     messages: List,
     model: str,
     llm_provider: str,

litellm/litellm_core_utils/realtime_streaming.py

@@ -1198,7 +1198,7 @@ class RealTimeStreaming:
         item["content"] = new_content
         return item
 
-    async def client_ack_messages(self):  # noqa: PLR0915
+    async def client_ack_messages(self):
         try:
             while True:
                 message = await self.websocket.receive_text()

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -209,7 +209,7 @@ class ChunkProcessor:
         )
         return response
 
-    def get_combined_tool_content(  # noqa: PLR0915
+    def get_combined_tool_content(
         self, tool_call_chunks: List[Dict[str, Any]]
     ) -> List[ChatCompletionMessageToolCall]:
         tool_calls_list: List[ChatCompletionMessageToolCall] = []
@@ -604,6 +604,8 @@ class ChunkProcessor:
                 usage_chunk = chunk._hidden_params.get("usage", None)
 
             if usage_chunk is not None:
+                if isinstance(usage_chunk, dict):
+                    usage_chunk = Usage(**usage_chunk)
                 usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
                 if (
                     usage_chunk_dict["prompt_tokens"] is not None

litellm/litellm_core_utils/streaming_handler.py

@@ -92,7 +92,7 @@ def is_async_iterable(obj: Any) -> bool:
 def print_verbose(print_statement):
     try:
         if litellm.set_verbose:
-            print(print_statement)  # noqa
+            print(print_statement)  # noqa: T201
     except Exception:
         pass
 
@@ -295,6 +295,12 @@ class CustomStreamWrapper:
         if len(self.chunks) < 2:
             return
 
+        # Providers like Vertex Gemini (Flash / Flash Lite with web search) emit
+        # metadata-only / usage-only chunks with no choices. These get stored in
+        # self.chunks but carry no comparable content, so skip repetition detection.
+        if not self.chunks[-1].choices or not self.chunks[-2].choices:
+            return
+
         last_content = self.chunks[-1].choices[0].delta.content
 
         if (
@@ -961,7 +967,7 @@ class CustomStreamWrapper:
                     delta, model_response.choices[0].delta, attribute
                 )
 
-    def return_processed_chunk_logic(  # noqa
+    def return_processed_chunk_logic(  # noqa: C901
         self,
         completion_obj: Dict[str, Any],
         model_response: ModelResponseStream,
@@ -1139,7 +1145,7 @@ class CustomStreamWrapper:
                 del model_response.choices[0].delta.reasoning_content
         return
 
-    def chunk_creator(self, chunk: Any):  # type: ignore  # noqa: PLR0915
+    def chunk_creator(self, chunk: Any):  # type: ignore
         if hasattr(chunk, "id"):
             self.response_id = chunk.id
         model_response = self.model_response_creator()
@@ -1881,7 +1887,7 @@ class CustomStreamWrapper:
             model_response.choices[0].finish_reason = "tool_calls"
         return model_response
 
-    def __next__(self) -> "ModelResponseStream":  # noqa: PLR0915
+    def __next__(self) -> "ModelResponseStream":
         cache_hit = False
         if (
             self.custom_llm_provider is not None
@@ -2071,7 +2077,7 @@ class CustomStreamWrapper:
 
         return self.completion_stream
 
-    async def __anext__(self) -> "ModelResponseStream":  # noqa: PLR0915
+    async def __anext__(self) -> "ModelResponseStream":
         cache_hit = False
         if (
             self.custom_llm_provider is not None

litellm/llms/anthropic/chat/handler.py

@@ -772,7 +772,7 @@ class ModelResponseIterator:
             )
         return results
 
-    def chunk_parser(self, chunk: dict) -> ModelResponseStream:  # noqa: PLR0915
+    def chunk_parser(self, chunk: dict) -> ModelResponseStream:
         try:
             type_chunk = chunk.get("type", "") or ""
 

litellm/llms/anthropic/chat/transformation.py

@@ -605,7 +605,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 )
         return _tool_choice
 
-    def _map_tool_helper(  # noqa: PLR0915
+    def _map_tool_helper(
         self,
         tool: ChatCompletionToolParam,
     ) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
@@ -1399,7 +1399,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
         return None
 
-    def map_openai_params(  # noqa: PLR0915
+    def map_openai_params(
         self,
         non_default_params: dict,
         optional_params: dict,
@@ -2214,18 +2214,33 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if "inference_geo" in _usage and _usage["inference_geo"] is not None:
             inference_geo = _usage["inference_geo"]
 
-        if (
-            "cache_creation_input_tokens" in _usage
-            and _usage["cache_creation_input_tokens"] is not None
-        ):
-            cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
-            prompt_tokens += cache_creation_input_tokens
-        if (
-            "cache_read_input_tokens" in _usage
-            and _usage["cache_read_input_tokens"] is not None
-        ):
-            cache_read_input_tokens = _usage["cache_read_input_tokens"]
-            prompt_tokens += cache_read_input_tokens
+        iterations: Optional[List[Any]] = _usage.get("iterations")
+        if iterations:
+            prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations)
+            completion_tokens = sum(
+                it.get("output_tokens", 0) or 0 for it in iterations
+            )
+            cache_creation_input_tokens = sum(
+                it.get("cache_creation_input_tokens", 0) or 0 for it in iterations
+            )
+            cache_read_input_tokens = sum(
+                it.get("cache_read_input_tokens", 0) or 0 for it in iterations
+            )
+            prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens
+
+        if not iterations:
+            if (
+                "cache_creation_input_tokens" in _usage
+                and _usage["cache_creation_input_tokens"] is not None
+            ):
+                cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
+                prompt_tokens += cache_creation_input_tokens
+            if (
+                "cache_read_input_tokens" in _usage
+                and _usage["cache_read_input_tokens"] is not None
+            ):
+                cache_read_input_tokens = _usage["cache_read_input_tokens"]
+                prompt_tokens += cache_read_input_tokens
         if "server_tool_use" in _usage and _usage["server_tool_use"] is not None:
             if (
                 "web_search_requests" in _usage["server_tool_use"]
@@ -2264,7 +2279,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 ),
             )
 
-        raw_input_tokens = usage_object.get("input_tokens", 0) or 0
+        raw_input_tokens = (
+            prompt_tokens - cache_read_input_tokens - cache_creation_input_tokens
+        )
         prompt_tokens_details = PromptTokensDetailsWrapper(
             cached_tokens=cache_read_input_tokens,
             cache_creation_tokens=cache_creation_input_tokens,
@@ -2296,6 +2313,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             cache_creation_input_tokens=cache_creation_input_tokens,
             cache_read_input_tokens=cache_read_input_tokens,
             completion_tokens_details=completion_token_details,
+            iterations=iterations,
             server_tool_use=(
                 ServerToolUse(
                     web_search_requests=web_search_requests,

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -372,7 +372,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             cache_read_input_tokens=0,
         )
 
-    def __next__(self):  # noqa: PLR0915
+    def __next__(self):
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
         try:
@@ -618,7 +618,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             )
             raise StopIteration
 
-    async def __anext__(self):  # noqa: PLR0915
+    async def __anext__(self):
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
         try:

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -332,7 +332,14 @@ class LiteLLMAnthropicMessagesAdapter:
             if isinstance(source, dict)
             else getattr(source, "cache_control", None)
         )
-        if cache_control and model and self.is_anthropic_claude_model(model):
+        if (
+            cache_control
+            and model
+            and (
+                self.is_anthropic_claude_model(model)
+                or self.is_bedrock_arn_model(model)
+            )
+        ):
             # TypedDict objects support dict operations at runtime
             # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
             if isinstance(target, dict):
@@ -376,7 +383,7 @@ class LiteLLMAnthropicMessagesAdapter:
             isinstance(tool_type, str) and tool_type.startswith("web_search")
         ) or tool_name == "web_search"
 
-    def translate_anthropic_messages_to_openai(  # noqa: PLR0915
+    def translate_anthropic_messages_to_openai(
         self,
         messages: List[
             Union[
@@ -752,6 +759,20 @@ class LiteLLMAnthropicMessagesAdapter:
         model_lower = model.lower()
         return "anthropic" in model_lower or "claude" in model_lower
 
+    @staticmethod
+    def is_bedrock_arn_model(model: str) -> bool:
+        """
+        Check if the model string is a Bedrock ARN, such as an Application
+        Inference Profile (e.g. arn:aws:bedrock:us-east-1:123:application-inference-profile/id).
+
+        These ARNs contain neither "anthropic" nor "claude", so is_anthropic_claude_model
+        cannot identify them even though, on the /v1/messages endpoint, they point at Claude.
+        Match ":bedrock:" in the ARN service field so another service's ARN that merely names
+        bedrock in a resource (arn:aws:sagemaker:.../my-bedrock-endpoint) is not matched.
+        """
+        model_lower = model.lower()
+        return "arn:" in model_lower and ":bedrock:" in model_lower
+
     @staticmethod
     def translate_thinking_for_model(
         thinking: Dict[str, Any],

litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py

@@ -97,7 +97,7 @@ def _read_summary_max_tokens_setting() -> int:
     return COMPACT_SUMMARY_MAX_TOKENS
 
 
-async def _check_summary_model_access(  # noqa: PLR0915
+async def _check_summary_model_access(
     user_api_key_auth: Any,
     summary_model: str,
     llm_router: Any,
@@ -970,7 +970,7 @@ def apply_client_compaction_block_history(
     )
 
 
-async def apply_compact_20260112(  # noqa: PLR0915
+async def apply_compact_20260112(
     *,
     model: str,
     messages: List[Dict[str, Any]],

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -66,7 +66,7 @@ class AnthropicResponsesStreamWrapper:
         self._current_block_index += 1
         return self._current_block_index
 
-    def _process_event(self, event: Any) -> None:  # noqa: PLR0915
+    def _process_event(self, event: Any) -> None:
         """Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
         event_type = getattr(event, "type", None)
         if event_type is None and isinstance(event, dict):

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -51,7 +51,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             return source.get("url")
         return None
 
-    def translate_messages_to_responses_input(  # noqa: PLR0915
+    def translate_messages_to_responses_input(
         self,
         messages: List[
             Union[

litellm/llms/azure/azure.py

@@ -189,7 +189,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except Exception as e:
             raise e
 
-    def completion(  # noqa: PLR0915
+    def completion(
         self,
         model: str,
         messages: list,

litellm/llms/azure/completion/handler.py

@@ -25,7 +25,7 @@ class AzureTextCompletion(BaseAzureLLM):
             headers["Authorization"] = f"Bearer {azure_ad_token}"
         return headers
 
-    def completion(  # noqa: PLR0915
+    def completion(
         self,
         model: str,
         messages: list,

litellm/llms/base_llm/base_model_iterator.py

@@ -1,8 +1,11 @@
 import json
 from abc import abstractmethod
-from typing import List, Optional, Union, cast
+from typing import TYPE_CHECKING, List, Optional, Union, cast
 
 import litellm
+
+if TYPE_CHECKING:
+    import httpx
 from litellm.types.utils import (
     Choices,
     Delta,
@@ -69,6 +72,18 @@ class BaseModelResponseIterator:
         self.streaming_response = streaming_response
         self.response_iterator = self.streaming_response
         self.json_mode = json_mode
+        self.http_response: Optional["httpx.Response"] = None
+
+    async def aclose(self) -> None:
+        """Close the upstream HTTP response so the provider connection is
+        released (and a backend like vLLM aborts generation) when the stream
+        is abandoned before its natural end.
+
+        ``streaming_response`` is usually a bare ``aiter_lines()`` generator
+        that holds no reference to the response, so the handler that owns the
+        response attaches it here after construction."""
+        if self.http_response is not None:
+            await self.http_response.aclose()
 
     def chunk_parser(
         self, chunk: dict

litellm/llms/bedrock/chat/converse_handler.py

@@ -248,7 +248,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             encoding=encoding,
         )
 
-    def completion(  # noqa: PLR0915
+    def completion(
         self,
         model: str,
         messages: list,

litellm/llms/bedrock/chat/converse_transformation.py

@@ -2189,7 +2189,7 @@ class AmazonConverseConfig(BaseConfig):
         real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
         return real_tools if real_tools else None
 
-    def _transform_response(  # noqa: PLR0915
+    def _transform_response(
         self,
         model: str,
         response: httpx.Response,

litellm/llms/bedrock/chat/invoke_handler.py

@@ -473,7 +473,7 @@ class BedrockLLM(BaseAWSLLM):
                     prompt += f"{message['content']}"
         return prompt, chat_history  # type: ignore
 
-    def process_response(  # noqa: PLR0915
+    def process_response(
         self,
         model: str,
         response: httpx.Response,
@@ -765,7 +765,7 @@ class BedrockLLM(BaseAWSLLM):
 
         return model_response
 
-    def completion(  # noqa: PLR0915
+    def completion(
         self,
         model: str,
         messages: list,

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -257,7 +257,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
 
         return request_data
 
-    def transform_response(  # noqa: PLR0915
+    def transform_response(
         self,
         model: str,
         raw_response: httpx.Response,

litellm/llms/bedrock/embed/embedding.py

@@ -388,7 +388,7 @@ class BedrockEmbedding(BaseAWSLLM):
             batch_data=batch_data,
         )
 
-    def embeddings(  # noqa: PLR0915
+    def embeddings(
         self,
         model: str,
         input: List[str],

litellm/llms/bedrock/image_edit/stability_transformation.py

@@ -149,7 +149,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
 
         return mapped_params
 
-    def transform_image_edit_request(  # noqa: PLR0915
+    def transform_image_edit_request(
         self,
         model: str,
         prompt: Optional[str],

litellm/llms/bedrock/passthrough/guardrail_translation/handler.py

@@ -226,7 +226,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
         return _is_converse_endpoint(endpoint)
 
     @staticmethod
-    async def de_anonymize_event_stream(  # noqa: PLR0915
+    async def de_anonymize_event_stream(
         body_bytes: bytes,
         proxy_logging_obj: "ProxyLogging",
         user_api_key_dict: "UserAPIKeyAuth",

litellm/llms/bytez/chat/transformation.py

@@ -191,7 +191,7 @@ class BytezChatConfig(BaseConfig):
         api_key: Optional[str] = None,
         json_mode: Optional[bool] = None,
     ) -> ModelResponse:
-        json = raw_response.json()  # noqa: F811
+        json = raw_response.json()
 
         error = json.get("error")
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -33,7 +33,10 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
 from litellm.llms.base_llm.audio_transcription.transformation import (
     BaseAudioTranscriptionConfig,
 )
-from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
+from litellm.llms.base_llm.base_model_iterator import (
+    BaseModelResponseIterator,
+    MockResponseIterator,
+)
 from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
 from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
@@ -814,6 +817,8 @@ class BaseLLMHTTPHandler:
             completion_stream = provider_config.get_model_response_iterator(
                 streaming_response=response.aiter_lines(), sync_stream=False
             )
+            if isinstance(completion_stream, BaseModelResponseIterator):
+                completion_stream.http_response = response
         # LOGGING
         logging_obj.post_call(
             input=messages,
@@ -5671,7 +5676,7 @@ class BaseLLMHTTPHandler:
                 )
             raise
 
-    async def async_responses_websocket(  # noqa: PLR0915
+    async def async_responses_websocket(
         self,
         model: str,
         websocket: Any,

litellm/llms/fastcrw/__init__.py

@@ -0,0 +1,7 @@
+"""
+fastCRW API integration module.
+"""
+
+from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig
+
+__all__ = ["FastCRWSearchConfig"]

litellm/llms/fastcrw/search/__init__.py

@@ -0,0 +1,7 @@
+"""
+fastCRW Search API module.
+"""
+
+from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig
+
+__all__ = ["FastCRWSearchConfig"]

litellm/llms/fastcrw/search/transformation.py

@@ -0,0 +1,182 @@
+"""
+Calls fastCRW's /v1/search endpoint to search the web.
+
+fastCRW is a Firecrawl-compatible web data engine (single Rust binary; self-host
+or cloud). The search response uses the Firecrawl-compatible envelope
+{ "success": true, "data": [ { "title", "url", "description", "markdown"? } ] }.
+
+fastCRW API Reference: https://fastcrw.com/docs/rest-api
+"""
+
+from typing import Optional, TypedDict, Union
+
+import httpx
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.search.transformation import (
+    BaseSearchConfig,
+    SearchResponse,
+    SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+
+class _FastCRWSearchRequestRequired(TypedDict):
+    """Required fields for fastCRW Search API request."""
+
+    query: str  # Required - search query
+
+
+class FastCRWSearchRequest(_FastCRWSearchRequestRequired, total=False):
+    """
+    fastCRW Search API request format.
+    Based on: https://fastcrw.com/docs/rest-api
+    """
+
+    limit: int  # Optional - maximum number of results to return
+    sources: list[
+        str
+    ]  # Optional - sources to search ('web', 'images'), default ['web']
+    scrapeOptions: dict  # Optional - options for scraping search results
+
+
+class FastCRWSearchConfig(BaseSearchConfig):
+    FASTCRW_API_BASE = "https://fastcrw.com/api/v1"
+
+    @staticmethod
+    def ui_friendly_name() -> str:
+        return "fastCRW"
+
+    def validate_environment(
+        self,
+        headers: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        **kwargs,
+    ) -> dict:
+        """
+        Validate environment and return headers.
+        """
+        api_key = api_key or get_secret_str("CRW_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "CRW_API_KEY is not set. Set `CRW_API_KEY` environment variable."
+            )
+        headers["Authorization"] = f"Bearer {api_key}"
+        headers["Content-Type"] = "application/json"
+        return headers
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        optional_params: dict,
+        data: Optional[Union[dict, list[dict]]] = None,
+        **kwargs,
+    ) -> str:
+        """
+        Get complete URL for Search endpoint.
+        """
+        api_base = api_base or get_secret_str("CRW_API_BASE") or self.FASTCRW_API_BASE
+
+        # Append "/search" to the api base if it's not already there
+        if not api_base.endswith("/search"):
+            api_base = f"{api_base}/search"
+
+        return api_base
+
+    def transform_search_request(
+        self,
+        query: Union[str, list[str]],
+        optional_params: dict,
+        **kwargs,
+    ) -> dict:
+        """
+        Transform Search request to fastCRW API format.
+
+        Transforms Perplexity unified spec parameters:
+        - query -> query (same)
+        - max_results -> limit
+
+        All other fastCRW-specific parameters are passed through as-is.
+
+        Args:
+            query: Search query (string or list of strings). fastCRW only supports single string queries.
+            optional_params: Optional parameters for the request
+
+        Returns:
+            Dict with typed request data following FastCRWSearchRequest spec
+        """
+        if isinstance(query, list):
+            # fastCRW only supports single string queries, join with spaces
+            query = " ".join(query)
+
+        request_data: FastCRWSearchRequest = {
+            "query": query,
+        }
+
+        # Transform Perplexity unified spec parameters to fastCRW format
+        if "max_results" in optional_params:
+            request_data["limit"] = optional_params["max_results"]
+
+        # Convert to dict before dynamic key assignments
+        result_data = dict(request_data)
+
+        # pass through all other parameters as-is
+        for param, value in optional_params.items():
+            if (
+                param not in self.get_supported_perplexity_optional_params()
+                and param not in result_data
+            ):
+                result_data[param] = value
+
+        # By default, request markdown content if not explicitly specified
+        # fastCRW doesn't return content unless explicitly requested via scrapeOptions
+        if "scrapeOptions" not in result_data:
+            result_data["scrapeOptions"] = {
+                "formats": ["markdown"],
+                "onlyMainContent": True,
+            }
+
+        return result_data
+
+    def transform_search_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        **kwargs,
+    ) -> SearchResponse:
+        """
+        Transform fastCRW API response to LiteLLM unified SearchResponse format.
+
+        fastCRW (Firecrawl-compatible) returns:
+            {"success": true, "data": [{"url": "...", "title": "...", "description": "...", "markdown"?: "..."}, ...]}
+
+        Args:
+            raw_response: Raw httpx response from fastCRW API
+            logging_obj: Logging object for tracking
+
+        Returns:
+            SearchResponse with standardized format
+        """
+        response_json = raw_response.json()
+
+        results = []
+
+        data = response_json.get("data", [])
+
+        if isinstance(data, list):
+            for result in data:
+                snippet = result.get("markdown") or result.get("description", "")
+                search_result = SearchResult(
+                    title=result.get("title", ""),
+                    url=result.get("url", ""),
+                    snippet=snippet,
+                    date=None,
+                    last_updated=None,
+                )
+                results.append(search_result)
+
+        return SearchResponse(
+            results=results,
+            object="search",
+        )

litellm/llms/gemini/realtime/transformation.py

@@ -1378,7 +1378,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             raise ValueError(f"Unknown openai event: {key}, value: {value}")
         return openai_event
 
-    def transform_realtime_response(  # noqa: PLR0915
+    def transform_realtime_response(
         self,
         message: Union[str, bytes],
         model: str,

litellm/llms/huggingface/embedding/transformation.py

@@ -404,7 +404,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
         )
         return completion_response
 
-    def convert_to_model_response_object(  # noqa: PLR0915
+    def convert_to_model_response_object(
         self,
         completion_response: Union[List[Dict[str, Any]], Dict[str, Any]],
         model_response: ModelResponse,

litellm/llms/modelscope/chat/transformation.py

@@ -0,0 +1,93 @@
+"""
+Translates from OpenAI's `/v1/chat/completions` to ModelScope's `/v1/chat/completions`
+"""
+
+from typing import Any, Coroutine, Literal, Optional, Tuple, Union, cast, overload
+
+from typing_extensions import override
+
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+def _has_non_text_content(message: AllMessageValues) -> bool:
+    """Check if a message has non-text content items (e.g. image_url)."""
+    content = message.get("content")
+    if not isinstance(content, list):
+        return False
+    return any(item.get("type") != "text" for item in content)
+
+
+class ModelScopeChatConfig(OpenAIGPTConfig):
+    DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1"
+
+    @overload
+    def _transform_messages(
+        self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
+    ) -> Coroutine[Any, Any, list[AllMessageValues]]: ...
+
+    @overload
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],
+        model: str,
+        is_async: Literal[False] = False,
+    ) -> list[AllMessageValues]: ...
+
+    def _transform_messages(
+        self, messages: list[AllMessageValues], model: str, is_async: bool = False
+    ) -> Union[list[AllMessageValues], Coroutine[Any, Any, list[AllMessageValues]]]:
+        """
+        Flatten text-only content lists to strings for ModelScope.
+
+        Messages with non-text content (e.g. image_url for vision models)
+        are kept as lists so the parent class can normalize them properly.
+        """
+        messages = [cast(AllMessageValues, {**m}) for m in messages]
+        for message in messages:
+            if _has_non_text_content(message):
+                continue
+            content = message.get("content")
+            if isinstance(content, list):
+                message["content"] = "".join(item.get("text") or "" for item in content)
+
+        if is_async:
+            return super()._transform_messages(
+                messages=messages, model=model, is_async=True
+            )
+        else:
+            return super()._transform_messages(
+                messages=messages, model=model, is_async=False
+            )
+
+    def _get_openai_compatible_provider_info(
+        self, api_base: Optional[str], api_key: Optional[str]
+    ) -> Tuple[Optional[str], Optional[str]]:
+        api_base = (
+            api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL
+        )  # type: ignore
+        dynamic_api_key = api_key or get_secret_str("MODELSCOPE_API_KEY")
+        return api_base, dynamic_api_key
+
+    @override
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        """
+        If api_base is not provided, use the default ModelScope /chat/completions endpoint.
+        """
+        if not api_base:
+            api_base = self.DEFAULT_BASE_URL
+
+        if not api_base.endswith("/chat/completions"):
+            api_base = f"{api_base}/chat/completions"
+
+        return api_base

litellm/llms/modelscope/image_generation/__init__.py

@@ -0,0 +1,31 @@
+"""
+ModelScope Image Generation Module
+
+Factory function for getting the appropriate config class.
+"""
+
+from litellm.llms.base_llm.image_generation.transformation import (
+    BaseImageGenerationConfig,
+)
+
+from .transformation import ModelScopeImageGenerationConfig
+
+__all__ = [
+    "ModelScopeImageGenerationConfig",
+    "get_modelscope_image_generation_config",
+]
+
+
+def get_modelscope_image_generation_config(
+    model: str,
+) -> BaseImageGenerationConfig:
+    """
+    Get the ModelScope config for image generation.
+
+    Args:
+        model: The model name (e.g., "modelscope/Qwen/Qwen-Image-Edit")
+
+    Returns:
+        BaseImageGenerationConfig instance for ModelScope
+    """
+    return ModelScopeImageGenerationConfig()

litellm/llms/modelscope/image_generation/transformation.py

@@ -0,0 +1,248 @@
+"""
+ModelScope Image Generation Config
+
+Handles transformation between OpenAI-compatible format and ModelScope API format.
+
+API Reference: https://modelscope.cn/docs/model-service/API-Inference/intro
+"""
+
+from typing import TYPE_CHECKING, Optional, Union
+
+import httpx
+from typing_extensions import override
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.image_generation.transformation import (
+    BaseImageGenerationConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIImageGenerationOptionalParams,
+)
+from litellm.types.utils import ImageObject, ImageResponse
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+    LiteLLMLoggingObj = object
+
+
+class ModelScopeImageGenerationConfig(BaseImageGenerationConfig):
+    """
+    Configuration for ModelScope image generation.
+
+    Supports text-to-image models like:
+    - Qwen/Qwen-Image-Edit
+    - And other ModelScope-hosted image generation models
+    """
+
+    DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1"
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIImageGenerationOptionalParams]:
+        """
+        Return list of OpenAI params supported by ModelScope.
+
+        ModelScope supports standard OpenAI image generation parameters.
+        """
+        return [
+            "n",  # Number of images to generate
+            "size",  # Size of the generated images
+            "response_format",  # url or b64_json
+            "user",  # User identifier
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        """
+        Map OpenAI parameters to ModelScope parameters.
+
+        ModelScope uses the same parameter names as OpenAI.
+        """
+        supported_params = self.get_supported_openai_params(model)
+        if drop_params:
+            non_default_params = {
+                k: v for k, v in non_default_params.items() if k in supported_params
+            }
+        optional_params.update(non_default_params)
+        return optional_params
+
+    @override
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        """
+        Get the complete URL for the ModelScope image generation API request.
+        """
+        base_url: str = (
+            api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL
+        )
+        base_url = base_url.rstrip("/")
+
+        # Return the images endpoint
+        return f"{base_url}/images/generations"
+
+    @override
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: list[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """
+        Validate environment and set up headers for ModelScope.
+        """
+        final_api_key: Optional[str] = api_key or get_secret_str("MODELSCOPE_API_KEY")
+
+        if not final_api_key:
+            raise ValueError(
+                "MODELSCOPE_API_KEY is not set. "
+                "Please set it via environment variable or pass api_key parameter."
+            )
+
+        default_headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {final_api_key}",
+        }
+
+        headers = {**headers, **default_headers}
+        return headers
+
+    def transform_image_generation_request(
+        self,
+        model: str,
+        prompt: str,
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Transform OpenAI-style request to ModelScope request format.
+
+        ModelScope uses the same format as OpenAI for image generation.
+        """
+        # Build the request body (same as OpenAI)
+        request_data: dict = {
+            "model": model,
+            "prompt": prompt,
+        }
+
+        # Add optional params
+        for key, value in optional_params.items():
+            if key.startswith("_"):
+                continue
+            request_data[key] = value
+
+        return request_data
+
+    @override
+    def transform_image_generation_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: ImageResponse,
+        logging_obj: LiteLLMLoggingObj,
+        request_data: dict,
+        optional_params: dict,
+        litellm_params: dict,
+        encoding: object,
+        api_key: Optional[str] = None,
+        json_mode: Optional[bool] = None,
+    ) -> ImageResponse:
+        """
+        Transform ModelScope response to OpenAI-compatible ImageResponse.
+
+        ModelScope returns the same format as OpenAI:
+        {"created": timestamp, "data": [{"url": "..."}]}
+        """
+        try:
+            response_data = raw_response.json()
+        except Exception as e:
+            raise self.get_error_class(
+                error_message=f"Error parsing ModelScope response: {e}",
+                status_code=raw_response.status_code,
+                headers=raw_response.headers,
+            )
+
+        # Check for errors in response
+        if "error" in response_data:
+            error_msg = response_data["error"].get(
+                "message", str(response_data["error"])
+            )
+            raise self.get_error_class(
+                error_message=f"ModelScope error: {error_msg}",
+                status_code=raw_response.status_code,
+                headers=raw_response.headers,
+            )
+
+        # Extract images from response
+        data_list = response_data.get("data", [])
+        if not model_response.data:
+            model_response.data = []
+
+        for item in data_list:
+            image_obj = ImageObject(
+                url=item.get("url"),
+                b64_json=item.get("b64_json"),
+                revised_prompt=item.get("revised_prompt"),
+            )
+            model_response.data.append(image_obj)
+
+        return model_response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: Union[dict, httpx.Headers],
+    ) -> BaseLLMException:
+        """Return the appropriate error class for ModelScope."""
+        from litellm.exceptions import (
+            AuthenticationError,
+            BadRequestError,
+            InternalServerError,
+        )
+
+        if status_code == 400:
+            return BadRequestError(  # type: ignore[return-value]
+                message=error_message,
+                model="",
+                llm_provider="modelscope",
+            )
+        elif status_code == 401:
+            return AuthenticationError(  # type: ignore[return-value]
+                message=error_message,
+                model="",
+                llm_provider="modelscope",
+            )
+        elif status_code >= 500:
+            return InternalServerError(  # type: ignore[return-value]
+                message=error_message,
+                model="",
+                llm_provider="modelscope",
+            )
+        else:
+            return BadRequestError(  # type: ignore[return-value]
+                message=error_message,
+                model="",
+                llm_provider="modelscope",
+            )

litellm/llms/openai/openai.py

@@ -608,7 +608,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
 
         return streaming_response
 
-    def completion(  # type: ignore # noqa: PLR0915
+    def completion(  # type: ignore
         self,
         model_response: ModelResponse,
         timeout: Union[float, httpx.Timeout],

litellm/llms/openai_like/providers.json

@@ -143,6 +143,14 @@
       "force_store_false": true
     }
   },
+  "libertai": {
+    "base_url": "https://api.libertai.io/v1",
+    "api_key_env": "LIBERTAI_API_KEY",
+    "api_base_env": "LIBERTAI_API_BASE",
+    "param_mappings": {
+      "max_completion_tokens": "max_tokens"
+    }
+  },
   "empiriolabs": {
     "base_url": "https://api.empiriolabs.ai/v1",
     "api_key_env": "EMPIRIOLABS_API_KEY",

litellm/llms/predibase/chat/transformation.py

@@ -129,7 +129,7 @@ class PredibaseConfig(BaseConfig):
                 optional_params["response_format"] = value
         return optional_params
 
-    def transform_response(  # noqa: PLR0915
+    def transform_response(
         self,
         model: str,
         raw_response: Response,

litellm/llms/sagemaker/completion/handler.py

@@ -138,7 +138,7 @@ class SagemakerLLM(BaseAWSLLM):
 
         return prepped_request
 
-    def completion(  # noqa: PLR0915
+    def completion(
         self,
         model: str,
         messages: list,

litellm/llms/vertex_ai/gemini/transformation.py

@@ -678,7 +678,7 @@ def check_if_part_exists_in_parts(
     return False
 
 
-def _gemini_convert_messages_with_history(  # noqa: PLR0915
+def _gemini_convert_messages_with_history(
     messages: List[AllMessageValues],
     model: Optional[str] = None,
     litellm_params: Optional[dict] = None,
@@ -1176,7 +1176,7 @@ def _rewrite_google_maps_response_format(data: RequestBody) -> None:
         _rewrite_mime_type_to_response_format(generation_config)
 
 
-def _transform_request_body(  # noqa: PLR0915
+def _transform_request_body(
     messages: List[AllMessageValues],
     model: str,
     optional_params: dict,

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -614,9 +614,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return googleSearch, googleSearchRetrieval, enterpriseWebSearch, urlContext
 
-    def _map_function(  # noqa: PLR0915
-        self, value: List[dict], optional_params: dict
-    ) -> List[Tools]:
+    def _map_function(self, value: List[dict], optional_params: dict) -> List[Tools]:
         """
         Map OpenAI-style tools/functions to Vertex AI format.
 
@@ -1173,7 +1171,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 optional_params["include_server_side_tool_invocations"] = True
                 return
 
-    def map_openai_params(  # noqa: PLR0915
+    def map_openai_params(
         self,
         non_default_params: Dict,
         optional_params: Dict,
@@ -1904,7 +1902,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             return False
 
     @staticmethod
-    def _calculate_usage(  # noqa: PLR0915
+    def _calculate_usage(
         completion_response: Union[
             GenerateContentResponseBody, BidiGenerateContentServerMessage
         ],
@@ -2380,7 +2378,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         return annotations
 
     @staticmethod
-    def _process_candidates(  # noqa: PLR0915
+    def _process_candidates(
         _candidates: List[Candidates],
         model_response: Union[ModelResponse, "ModelResponseStream"],
         standard_optional_params: dict,

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py

@@ -124,7 +124,7 @@ class GoogleBatchEmbeddings(VertexLLM):
 
         return resolved_files
 
-    def batch_embeddings(  # noqa: PLR0915
+    def batch_embeddings(
         self,
         model: str,
         input: GeminiEmbeddingInput,

litellm/llms/vertex_ai/vertex_ai_non_gemini.py

@@ -77,7 +77,7 @@ def _set_client_in_cache(client_cache_key: str, vertex_llm_model: Any):
     )
 
 
-def completion(  # noqa: PLR0915
+def completion(
     model: str,
     messages: list,
     model_response: ModelResponse,
@@ -485,7 +485,7 @@ def completion(  # noqa: PLR0915
         )
 
 
-async def async_completion(  # noqa: PLR0915
+async def async_completion(
     llm_model,
     mode: str,
     prompt: str,
@@ -650,7 +650,7 @@ async def async_completion(  # noqa: PLR0915
         raise VertexAIError(status_code=500, message=str(e))
 
 
-async def async_streaming(  # noqa: PLR0915
+async def async_streaming(
     llm_model,
     mode: str,
     prompt: str,

litellm/main.py

@@ -392,7 +392,7 @@ class AsyncCompletions:
 
 @tracer.wrap()
 @client
-async def acompletion(  # noqa: PLR0915
+async def acompletion(
     model: str,
     # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
     messages: List = [],
@@ -1086,7 +1086,7 @@ def _build_custom_pricing_entry(
 
 @tracer.wrap()
 @client
-def completion(  # type: ignore # noqa: PLR0915
+def completion(  # type: ignore
     model: str,
     # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
     messages: List = [],
@@ -4878,7 +4878,7 @@ def embedding(
 
 
 @client
-def embedding(  # noqa: PLR0915
+def embedding(
     model,
     input=[],
     # Optional params
@@ -6125,7 +6125,7 @@ async def atext_completion(
 
 
 @client
-def text_completion(  # noqa: PLR0915
+def text_completion(
     prompt: Union[
         str, List[Union[str, List[Union[str, List[int]]]]]
     ],  # Required: The prompt(s) to generate completions for.
@@ -6664,7 +6664,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
 
 
 @client
-def transcription(  # noqa: PLR0915
+def transcription(
     model: str,
     file: FileTypes,
     ## OPTIONAL OPENAI PARAMS ##
@@ -6971,7 +6971,7 @@ async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent:
 
 
 @client
-def speech(  # noqa: PLR0915
+def speech(
     model: str,
     input: str,
     voice: Optional[Union[str, dict]] = None,
@@ -7572,7 +7572,7 @@ def print_verbose(print_statement):
     try:
         verbose_logger.debug(print_statement)
         if litellm.set_verbose:
-            print(print_statement)  # noqa
+            print(print_statement)  # noqa: T201
     except Exception:
         pass
 
@@ -7662,7 +7662,7 @@ def stream_chunk_builder_text_completion(
     return TextCompletionResponse(**response)
 
 
-def stream_chunk_builder(  # noqa: PLR0915
+def stream_chunk_builder(
     chunks: list,
     messages: Optional[list] = None,
     start_time=None,

litellm/model_prices_and_context_window_backup.json

@@ -18822,6 +18822,38 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
+    "github_copilot/mai-code-1-flash": {
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 7.5e-07,
+        "litellm_provider": "github_copilot",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 4.5e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true
+    },
+    "github_copilot/mai-code-1-flash-internal": {
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 7.5e-07,
+        "litellm_provider": "github_copilot",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 4.5e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true
+    },
     "github_copilot/text-embedding-3-small": {
         "litellm_provider": "github_copilot",
         "max_input_tokens": 8191,
@@ -40784,6 +40816,174 @@
         "litellm_provider": "llamagate",
         "mode": "embedding"
     },
+    "libertai/hermes-3-8b-tee": {
+        "max_tokens": 16000,
+        "max_input_tokens": 16000,
+        "max_output_tokens": 16000,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 6e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": false,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/gemma-4-31b-it": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 4e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/gemma-4-31b-it-thinking": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 4e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "supports_reasoning": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/qwen3.6-27b": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/qwen3.6-27b-thinking": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "supports_reasoning": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/qwen3.6-35b-a3b": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/qwen3.6-35b-a3b-thinking": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "supports_reasoning": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/qwen3.5-122b-a10b": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 1.75e-06,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/qwen3.5-122b-a10b-thinking": {
+        "max_tokens": 262144,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "input_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 1.75e-06,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "supports_reasoning": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/deepseek-v4-flash": {
+        "max_tokens": 200000,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 200000,
+        "input_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 1.75e-06,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": false,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/deepseek-v4-flash-thinking": {
+        "max_tokens": 200000,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 200000,
+        "input_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 1.75e-06,
+        "litellm_provider": "libertai",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_system_messages": true,
+        "supports_vision": false,
+        "supports_reasoning": true,
+        "source": "https://docs.libertai.io/apis/text/"
+    },
+    "libertai/bge-m3": {
+        "max_tokens": 8192,
+        "max_input_tokens": 8192,
+        "input_cost_per_token": 1e-08,
+        "output_cost_per_token": 0.0,
+        "litellm_provider": "libertai",
+        "mode": "embedding",
+        "source": "https://docs.libertai.io/apis/text/"
+    },
     "sarvam/sarvam-m": {
         "cache_creation_input_token_cost": 0,
         "cache_creation_input_token_cost_above_1hr": 0,

litellm/mypy.ini

@@ -1,13 +1,16 @@
 [mypy]
-warn_return_any = False
+warn_return_any = True
 ignore_missing_imports = True
+disallow_untyped_defs = True
 mypy_path = litellm/stubs
 namespace_packages = True
 disable_error_code =
-    valid-type,
     annotation-unchecked,
     import-untyped
 
+[mypy-litellm.*]
+ignore_missing_imports = False
+
 [mypy-google.*]
 ignore_missing_imports = True
 

litellm/provider_endpoints_support_backup.json

@@ -1288,6 +1288,23 @@
         "interactions": true
       }
     },
+    "libertai": {
+      "display_name": "LibertAI (`libertai`)",
+      "url": "https://docs.litellm.ai/docs/providers/libertai",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "litellm_proxy": {
       "display_name": "LiteLLM Proxy (`litellm_proxy`)",
       "url": "https://docs.litellm.ai/docs/providers/litellm_proxy",

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -67,9 +67,10 @@ def _is_mcp_passthrough_cold_start(
     spec-compliant WWW-Authenticate challenge instead of surfacing a generic
     admission error.
 
-    Uses "all" semantics (mirrors :meth:`MCPRequestHandler._target_servers_use_oauth2`):
-    one non-passthrough target in a co-targeted set must not flip the bypass
-    open for the others. Fails closed when any target cannot be resolved."""
+    Uses "all" semantics (mirrors
+    :meth:`MCPRequestHandler._target_servers_delegate_auth_to_upstream`): one
+    non-passthrough target in a co-targeted set must not flip the bypass open
+    for the others. Fails closed when any target cannot be resolved."""
     if not mcp_servers:
         return False
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
@@ -124,7 +125,7 @@ class MCPRequestHandler:
     LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME = SpecialHeaders.mcp_access_groups.value
 
     @staticmethod
-    async def process_mcp_request(  # noqa: PLR0915
+    async def process_mcp_request(
         scope: Scope,
     ) -> Tuple[
         UserAPIKeyAuth,
@@ -214,101 +215,64 @@ class MCPRequestHandler:
         # Only OAuth metadata routes registered under /.well-known/ are public.
         if request_route.startswith("/.well-known/"):
             validated_user_api_key_auth = UserAPIKeyAuth()
-        elif (
-            not litellm_api_key
-            and MCPRequestHandler._target_servers_delegate_auth_to_upstream(  # noqa: E501
-                path=request_route,
-                mcp_servers=mcp_servers,
-                client_ip=IPAddressUtils.get_mcp_client_ip(request),
-            )
-        ):
-            # Operator opted this oauth2 server into upstream-delegated auth
-            # (PKCE passthrough): skip LiteLLM API-key/SSO entirely so the
-            # client authenticates directly with the upstream MCP server.
-            # Fires ONLY when neither x-litellm-api-key nor Authorization is
-            # present. If any LiteLLM key is supplied (primary or secondary
-            # header), we fall through so user_id is resolved, spend/rate
-            # limiting apply, and any stored OAuth token can be retrieved
-            # and forwarded upstream. Gated by
-            # _target_servers_delegate_auth_to_upstream, which only returns
-            # True when EVERY target is auth_type=oauth2 AND has the
-            # delegate_auth_to_upstream flag set — fails closed otherwise.
-            validated_user_api_key_auth = UserAPIKeyAuth()
         elif has_explicit_litellm_key:
-            # Explicit x-litellm-api-key provided - always validate normally
+            # An explicit x-litellm-api-key is always a LiteLLM credential, even
+            # for a delegated server, so validate it: identity / spend / rate
+            # limits resolve and any stored upstream token can be forwarded.
             validated_user_api_key_auth = await user_api_key_auth(
                 api_key=litellm_api_key, request=request
             )
+        elif MCPRequestHandler._target_servers_delegate_auth_to_upstream(
+            path=request_route,
+            mcp_servers=mcp_servers,
+            client_ip=IPAddressUtils.get_mcp_client_ip(request),
+        ):
+            # Operator opted this oauth2 server into upstream-delegated auth: the
+            # client authenticates directly with the upstream MCP server, so any
+            # Authorization bearer is an upstream token, never a LiteLLM key. Skip
+            # LiteLLM validation entirely — covering both the no-credential
+            # discovery request and the authenticated call carrying the upstream
+            # bearer — so a tool call that succeeds never carries a phantom 401
+            # auth span; the bearer is forwarded upstream unchanged. Gated by
+            # _target_servers_delegate_auth_to_upstream, which returns True only
+            # when EVERY target is auth_type=oauth2 with delegate_auth_to_upstream
+            # set; fails closed otherwise.
+            validated_user_api_key_auth = UserAPIKeyAuth()
         elif oauth2_headers:
-            # No x-litellm-api-key, but Authorization header present.
-            # Could be a LiteLLM key (backward compat) OR an opaque OAuth2 token
-            # the operator wants forwarded to an upstream OAuth2-mode MCP server.
-            # Try LiteLLM auth first; on auth failure, only fall back to anonymous
-            # passthrough when the request actually targets a server whose operator
-            # configured ``auth_type=oauth2``. For any other server (api_key,
-            # bearer_token, basic, etc.), a failed LiteLLM auth is a real failure
-            # and must propagate — otherwise an attacker can exchange any garbage
-            # bearer for an anonymous session.
+            # Authorization on a non-delegated server: the bearer must be a real
+            # LiteLLM credential, so a failed validation is a genuine 401/403 and
+            # propagates. The sole anonymous fallback is the auth_type=none
+            # pass-through cold-start (RFC 9728 discovery return), gated on a 401
+            # so a recognized-but-forbidden key still fails closed.
+            client_ip = IPAddressUtils.get_mcp_client_ip(request)
             try:
                 validated_user_api_key_auth = await user_api_key_auth(
                     api_key=litellm_api_key, request=request
                 )
             except (HTTPException, ProxyException) as e:
-                # HTTPException.status_code is int; ProxyException.code is
-                # normalized to str in its __init__ but can be ``"None"`` or any
-                # non-numeric string when the caller didn't supply a numeric
-                # code, so we compare against both int and str forms rather
-                # than coercing (``int("None")`` would raise ValueError and
-                # rewrite the auth error as a 500).
+                # ProxyException.code is normalized to str (possibly "None"), so
+                # compare both int and str forms rather than coercing.
                 status = e.status_code if isinstance(e, HTTPException) else e.code
-                is_auth_error = status in (401, 403, "401", "403")
                 is_unauthenticated = status in (401, "401")
-                client_ip = IPAddressUtils.get_mcp_client_ip(request)
-                if is_auth_error and MCPRequestHandler._target_servers_use_oauth2(
-                    path=request_route,
-                    mcp_servers=mcp_servers,
-                    client_ip=client_ip,
+                mcp_servers_from_path = _parse_mcp_server_names_from_path(
+                    request_route, mcp_servers
+                )
+                if (
+                    is_unauthenticated
+                    and mcp_servers_from_path is not None
+                    and not _has_client_supplied_mcp_auth(
+                        mcp_auth_header,
+                        mcp_server_auth_headers,
+                    )
+                    and _is_mcp_passthrough_cold_start(
+                        mcp_servers_from_path, client_ip=client_ip
+                    )
                 ):
                     verbose_logger.debug(
-                        "MCP OAuth2: target server is OAuth2-mode, treating "
-                        "Authorization as upstream OAuth2 token passthrough"
+                        "MCP pass-through return: forwarding Authorization as "
+                        "upstream OAuth token for delegated auth"
                     )
                     validated_user_api_key_auth = UserAPIKeyAuth()
-                elif is_unauthenticated:
-                    # Pass-through cold-start return: per RFC 9728 / MCP
-                    # Authorization spec the client completes upstream OAuth
-                    # discovery and returns with ``Authorization: Bearer
-                    # <upstream-token>``. For ``auth_type=none`` passthrough
-                    # servers that bearer is not a LiteLLM key (auth above
-                    # failed) but is meant to be forwarded upstream
-                    # unchanged. Fall back to anonymous admission so the
-                    # caller is not rejected for following the discovery
-                    # flow without also setting ``x-litellm-api-key``.
-                    # Only trigger on 401 (token unrecognized); a 403 means
-                    # the key WAS recognized but is forbidden (e.g. over
-                    # budget / rate limited) and must propagate so those
-                    # controls are not bypassed via anonymous admission.
-                    mcp_servers_from_path = _parse_mcp_server_names_from_path(
-                        request_route, mcp_servers
-                    )
-                    if (
-                        mcp_servers_from_path is not None
-                        and not _has_client_supplied_mcp_auth(
-                            mcp_auth_header,
-                            mcp_server_auth_headers,
-                        )
-                        and _is_mcp_passthrough_cold_start(
-                            mcp_servers_from_path, client_ip=client_ip
-                        )
-                    ):
-                        verbose_logger.debug(
-                            "MCP pass-through return: target server is "
-                            "passthrough, treating Authorization as "
-                            "upstream OAuth token for delegated auth"
-                        )
-                        validated_user_api_key_auth = UserAPIKeyAuth()
-                    else:
-                        raise
                 else:
                     raise
         else:
@@ -412,45 +376,6 @@ class MCPRequestHandler:
             return [single_server_match.group(1)]
         return [servers_and_path]
 
-    @staticmethod
-    def _target_servers_use_oauth2(
-        path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str]
-    ) -> bool:
-        """
-        True only when EVERY MCP server the request targets is configured for
-        ``auth_type == oauth2``. If any target is non-OAuth2 — or if the target
-        cannot be resolved at all — return False so the caller fails closed.
-
-        Used to gate the "treat Authorization as opaque OAuth2 token" fallback
-        in :meth:`process_mcp_request` so a failed LiteLLM-auth cannot be
-        exchanged for an anonymous session against a non-OAuth2 server.
-        """
-        # Inline imports avoid a circular dependency: mcp_server_manager imports
-        # from this module.
-        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-            global_mcp_server_manager,
-        )
-        from litellm.types.mcp import MCPAuth
-
-        # Resolve the same target list downstream routing will use. For
-        # ``/mcp/...`` routes, ``extract_mcp_auth_context`` overrides the
-        # ``x-mcp-servers`` header with path-derived names, so we must mirror
-        # that here — otherwise a caller could set the header to a permissive
-        # server while the path targets a stricter one (header/path TOCTOU).
-        target_names = MCPRequestHandler._resolve_target_server_names(
-            path=path, mcp_servers_header=mcp_servers
-        )
-        if not target_names:
-            return False
-
-        for name in target_names:
-            server = global_mcp_server_manager.get_mcp_server_by_name(
-                name, client_ip=client_ip
-            )
-            if server is None or server.auth_type != MCPAuth.oauth2:
-                return False
-        return True
-
     @staticmethod
     def _target_servers_delegate_auth_to_upstream(
         path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str]
@@ -472,8 +397,8 @@ class MCPRequestHandler:
         )
         from litellm.types.mcp import MCPAuth
 
-        # See _target_servers_use_oauth2: must mirror the downstream
-        # header-vs-path override or an attacker could set
+        # Must mirror the downstream header-vs-path override
+        # (``extract_mcp_auth_context``) or an attacker could set
         # ``x-mcp-servers`` to a delegate-enabled server while the URL path
         # targets a non-delegate server, skipping LiteLLM auth for it.
         target_names = MCPRequestHandler._resolve_target_server_names(

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -3355,7 +3355,7 @@ class MCPServerManager:
             )
         )
 
-    async def _call_regular_mcp_tool(  # noqa: PLR0915
+    async def _call_regular_mcp_tool(
         self,
         mcp_server: MCPServer,
         original_tool_name: str,

litellm/proxy/_experimental/mcp_server/sampling_handler.py

@@ -661,7 +661,7 @@ def _convert_openai_response_to_mcp_result(
     )
 
 
-async def _check_model_access(  # noqa: PLR0915
+async def _check_model_access(
     model: str, user_api_key_auth: Any
 ) -> Optional["ErrorData"]:
     """Enforce model-permission checks for MCP sampling requests.

litellm/proxy/_experimental/mcp_server/server.py

@@ -617,7 +617,7 @@ if MCP_AVAILABLE:
                 active_mcp_session_var.reset(_session_reset_token)
 
     @server.call_tool()
-    async def mcp_server_tool_call(  # noqa: PLR0915
+    async def mcp_server_tool_call(
         name: str, arguments: Dict[str, Any] | None
     ) -> CallToolResult:
         """
@@ -1591,7 +1591,7 @@ if MCP_AVAILABLE:
             _mcp_gateway_initialize_instructions.reset(instructions_token)
             _mcp_gateway_server_name.reset(server_name_token)
 
-    async def _get_tools_from_mcp_servers(  # noqa: PLR0915
+    async def _get_tools_from_mcp_servers(
         user_api_key_auth: Optional[UserAPIKeyAuth],
         mcp_auth_header: Optional[str],
         mcp_servers: Optional[List[str]],
@@ -2435,7 +2435,7 @@ if MCP_AVAILABLE:
                 },
             )
 
-    async def execute_mcp_tool(  # noqa: PLR0915
+    async def execute_mcp_tool(
         name: str,
         arguments: Dict[str, Any],
         allowed_mcp_servers: List[MCPServer],
@@ -3642,7 +3642,7 @@ if MCP_AVAILABLE:
                     detail="Forbidden",
                 )
 
-    async def handle_streamable_http_mcp(  # noqa: PLR0915
+    async def handle_streamable_http_mcp(
         scope: Scope, receive: Receive, send: Send
     ) -> None:
         """Handle MCP requests through StreamableHTTP."""

litellm/proxy/_types.py

@@ -373,6 +373,8 @@ class LiteLLMRoutes(enum.Enum):
         # vector stores
         "/vector_stores",
         "/v1/vector_stores",
+        "/vector_stores/{vector_store_id}",
+        "/v1/vector_stores/{vector_store_id}",
         "/vector_stores/{vector_store_id}/search",
         "/v1/vector_stores/{vector_store_id}/search",
         "/vector_stores/{vector_store_id}/files",
@@ -2150,6 +2152,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
     master_key: Optional[str] = Field(
         None, description="require a key for all calls to proxy"
     )
+    allow_cli_sso_verification_uri_complete: bool | None = Field(
+        None,
+        description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine",
+    )
     database_url: Optional[str] = Field(
         None,
         description="connect to a postgres db - needed for generating temporary keys + tracking spend / key",