Merge branch 'main' into fix-deepeval-span-output

.circleci/config.yml

@@ -524,7 +524,15 @@ jobs:
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
+      - &postgres16_sidecar
+        image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
     working_directory: ~/project
+    environment: &sidecar_database_url
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
     parallelism: 4
     steps:
       - checkout
@@ -545,6 +553,9 @@ jobs:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
       - save_cargo_target
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -599,7 +610,9 @@ jobs:
   local_testing_part2:
     docker:
       - *python312_image
+      - *postgres16_sidecar
     working_directory: ~/project
+    environment: *sidecar_database_url
     parallelism: 4
     steps:
       - checkout
@@ -619,6 +632,9 @@ jobs:
           paths:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -673,7 +689,9 @@ jobs:
   langfuse_logging_unit_tests:
     docker:
       - *python312_image
+      - *postgres16_sidecar
     working_directory: ~/project
+    environment: *sidecar_database_url
     resource_class: medium
 
     steps:
@@ -694,6 +712,9 @@ jobs:
           paths:
             - ~/.cache/uv
           key: v1-uv-cache-{{ checksum "uv.lock" }}
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
       - run:
           name: Run prisma ./docker/entrypoint.sh
           command: |
@@ -722,14 +743,9 @@ jobs:
   auth_ui_unit_tests:
     docker:
       - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
+      - *postgres16_sidecar
     working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+    environment: *sidecar_database_url
 
     steps:
       - checkout
@@ -892,40 +908,6 @@ jobs:
           paths:
             - router_unit_tests_coverage.xml
             - router_unit_tests_coverage
-  litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
-    docker:
-      - *python312_image
-    working_directory: ~/project
-    resource_class: medium
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      # Run pytest and generate JUnit XML report
-      - setup_litellm_enterprise_pip
-      - run:
-          name: Run tests
-          command: |
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-                -v \
-                --junitxml=test-results/junit.xml \
-                --durations=5 \
-                -k \"assistants\""
-          no_output_timeout: 15m
-      # Store test results
-      - store_test_results:
-          path: test-results
   llm_translation_testing:
     docker:
       - *python312_image
@@ -1021,49 +1003,6 @@ jobs:
           paths:
             - realtime_translation_coverage.xml
             - realtime_translation_coverage
-  agent_testing:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      # Run pytest and generate JUnit XML report
-      - run:
-          name: Run tests
-          command: |
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-                -vv -s \
-                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-                --junitxml=test-results/junit.xml \
-                --durations=5"
-          no_output_timeout: 15m
-      - run:
-          name: Rename the coverage files
-          command: |
-            mv coverage.xml agent_coverage.xml
-            mv .coverage agent_coverage
-
-      # Store test results
-      - store_test_results:
-          path: test-results
-      - persist_to_workspace:
-          root: .
-          paths:
-            - agent_coverage.xml
-            - agent_coverage
   guardrails_testing:
     docker:
       - *python312_image
@@ -1547,7 +1486,9 @@ jobs:
   installing_litellm_on_python:
     docker:
       - *python312_image
+      - *postgres16_sidecar
     working_directory: ~/project
+    environment: *sidecar_database_url
 
     steps:
       - checkout
@@ -1560,6 +1501,9 @@ jobs:
           command: |
             uv sync --frozen --all-groups --all-extras --python 3.12
       - setup_litellm_enterprise_pip
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
       - run:
           name: Run tests
           command: |
@@ -1571,7 +1515,9 @@ jobs:
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
+      - *postgres16_sidecar
     working_directory: ~/project
+    environment: *sidecar_database_url
     resource_class: medium
 
     steps:
@@ -1584,6 +1530,9 @@ jobs:
           name: Install Dependencies
           command: |
             uv sync --frozen --all-groups --all-extras --python 3.13
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
       - run:
           name: Run tests
           command: |
@@ -1592,14 +1541,9 @@ jobs:
   installing_litellm_on_python_v2_migration_resolver:
     docker:
       - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
+      - *postgres16_sidecar
     working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+    environment: *sidecar_database_url
 
     steps:
       - checkout
@@ -2685,7 +2629,7 @@ jobs:
       - run:
           name: Combine Coverage
           command: |
-            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
+            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
             uv tool run --from 'coverage[toml]==7.10.6' coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -3063,6 +3007,11 @@ jobs:
       PYTHONPATH: tests/e2e
     steps:
       - checkout_migration_source
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - install_uv
       - install_rust
       - restore_cache:
@@ -3182,6 +3131,11 @@ jobs:
       E2E_FIXTURE_MODE: live
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes:
           category: provider-harness
       - setup_litellm_test_deps
@@ -3211,9 +3165,13 @@ jobs:
       parallelism:
         type: integer
         default: 1
+      resource_class:
+        type: enum
+        enum: [large, xlarge]
+        default: large
     machine:
       image: ubuntu-2204:2024.04.1
-    resource_class: large
+    resource_class: << parameters.resource_class >>
     working_directory: ~/project
     parallelism: << parameters.parallelism >>
     steps:
@@ -3556,7 +3514,11 @@ workflows:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, mcp, sdk, cost, security, browser]
+              suite: [management, accounting, database, providers, sdk, cost, security, browser]
+      - integration_contracts:
+          name: integration-mcp
+          suite: mcp
+          resource_class: xlarge
       - integration_contracts:
           name: integration-extensions
           suite: extensions
@@ -3614,7 +3576,6 @@ workflows:
       - local_testing_part1
       - local_testing_part2
       - langfuse_logging_unit_tests
-      - litellm_assistants_api_testing
       - litellm_router_testing
       - litellm_router_unit_testing
       - auth_ui_unit_tests
@@ -3648,7 +3609,6 @@ workflows:
             - build_docker_database_image
       - llm_translation_testing
       - realtime_translation_testing
-      - agent_testing
       - guardrails_testing
       - google_generate_content_endpoint_testing
       - llm_responses_api_testing
@@ -3663,7 +3623,6 @@ workflows:
       - upload-coverage:
           requires:
             - realtime_translation_testing
-            - agent_testing
             - google_generate_content_endpoint_testing
             - guardrails_testing
             - ocr_testing
@@ -3677,7 +3636,6 @@ workflows:
             - langfuse_logging_unit_tests
             - local_testing_part1
             - local_testing_part2
-            - litellm_assistants_api_testing
             - litellm_router_unit_testing
             - auth_ui_unit_tests
       - db_migration_disable_update_check:

.github/workflows/_test-unit-base.yml

@@ -3,6 +3,11 @@ name: _Unit Test Base (Reusable)
 on:
   workflow_call:
     inputs:
+      python-version:
+        description: "Python version used to install dependencies and run tests"
+        required: false
+        type: string
+        default: "3.12"
       test-path:
         description: >-
           Space-separated pytest paths to run. A path that no longer exists is
@@ -78,7 +83,7 @@ permissions:
   contents: read
 
 env:
-  UV_PYTHON: "3.12"
+  UV_PYTHON: ${{ inputs.python-version }}
   LITELLM_LOCAL_MODEL_COST_MAP: "True"
 
 jobs:

.github/workflows/test-unit-proxy-db.yml

@@ -39,6 +39,21 @@ concurrency:
 #     pinning the whole file to one worker (the default --dist=loadscope
 #     behavior for single-file targets).
 jobs:
+  lens-python-310:
+    name: Lens Python 3.10
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      python-version: "3.10"
+      test-path: tests/unit/proxy/lens/test_inference.py
+      workers: 0
+      reruns: 0
+      timeout-minutes: 5
+      artifact-name: lens-python-310
+
   # Fast guard — fails the workflow when a test directory or file inside a sharded
   # tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
   # so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list

enterprise/enterprise_hooks/__init__.py

@@ -1,15 +1,15 @@
 from typing import Dict, Literal, Type, Union
 
-from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
+from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles
 from litellm_enterprise.proxy.hooks.managed_vector_stores import (
-    _PROXY_LiteLLMManagedVectorStores,
+    PROXY_LiteLLMManagedVectorStores,
 )
 
 from litellm.integrations.custom_logger import CustomLogger
 
 ENTERPRISE_PROXY_HOOKS: Dict[str, Type[CustomLogger]] = {
-    "managed_files": _PROXY_LiteLLMManagedFiles,
-    "managed_vector_stores": _PROXY_LiteLLMManagedVectorStores,
+    "managed_files": PROXY_LiteLLMManagedFiles,
+    "managed_vector_stores": PROXY_LiteLLMManagedVectorStores,
 }
 
 

enterprise/enterprise_hooks/banned_keywords.py

@@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger
 from fastapi import HTTPException
 
 
-class _ENTERPRISE_BannedKeywords(CustomLogger):
+class ENTERPRISE_BannedKeywords(CustomLogger):
     enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self):
@@ -114,3 +114,4 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
         response: str,
     ):
         self.test_violation(test_str=response)
+_ENTERPRISE_BannedKeywords = ENTERPRISE_BannedKeywords

enterprise/enterprise_hooks/blocked_user_list.py

@@ -21,7 +21,7 @@ from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
 from litellm.proxy.utils import PrismaClient
 
 
-class _ENTERPRISE_BlockedUserList(CustomLogger):
+class ENTERPRISE_BlockedUserList(CustomLogger):
     enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self, prisma_client: Optional[PrismaClient]):
@@ -128,3 +128,4 @@ class _ENTERPRISE_BlockedUserList(CustomLogger):
                     str(e)
                 )
             )
+_ENTERPRISE_BlockedUserList = ENTERPRISE_BlockedUserList

enterprise/enterprise_hooks/google_text_moderation.py

@@ -16,7 +16,7 @@ from litellm.proxy.guardrails._content_utils import iter_message_text
 from litellm.types.utils import CallTypesLiteral
 
 
-class _ENTERPRISE_GoogleTextModeration(CustomLogger):
+class ENTERPRISE_GoogleTextModeration(CustomLogger):
     user_api_key_cache = None
     confidence_categories = [
         "toxic",
@@ -125,9 +125,10 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
                     )
             # Handle the response
             return data
+_ENTERPRISE_GoogleTextModeration = ENTERPRISE_GoogleTextModeration
 
 
-# google_text_moderation_obj = _ENTERPRISE_GoogleTextModeration()
+# google_text_moderation_obj = ENTERPRISE_GoogleTextModeration()
 # asyncio.run(
 #     google_text_moderation_obj.async_moderation_hook(
 #         data={"messages": [{"role": "user", "content": "Hey, how's it going?"}]}

enterprise/enterprise_hooks/openai_moderation.py

@@ -24,7 +24,7 @@ from litellm.proxy.guardrails._content_utils import iter_message_text
 from litellm.types.utils import CallTypesLiteral
 
 
-class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
+class ENTERPRISE_OpenAI_Moderation(CustomLogger):
     @property
     def model_name(self) -> str:
         return litellm.openai_moderations_model_name or DEFAULT_OPENAI_MODERATIONS_MODEL
@@ -55,3 +55,4 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
                 status_code=403, detail={"error": "Violated content safety policy"}
             )
         pass
+_ENTERPRISE_OpenAI_Moderation = ENTERPRISE_OpenAI_Moderation

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/endpoints.py

@@ -3,9 +3,10 @@ Endpoints for managing email alerts on litellm
 """
 
 import json
-from typing import Dict
+from typing import Dict, Final, cast
 
 from fastapi import APIRouter, Depends, HTTPException
+from pydantic import JsonValue
 from litellm_enterprise.types.enterprise_callbacks.send_emails import (
     DefaultEmailSettings,
     EmailEvent,
@@ -38,14 +39,15 @@ async def _get_email_settings(prisma_client) -> Dict[str, bool]:
             and general_settings_entry.param_value is not None
         ):
             # Get general settings value
-            if isinstance(general_settings_entry.param_value, str):
-                general_settings = json.loads(general_settings_entry.param_value)
-            else:
-                general_settings = general_settings_entry.param_value
+            general_settings: Final = (
+                cast(Dict[str, object], json.loads(general_settings_entry.param_value))
+                if isinstance(general_settings_entry.param_value, str)
+                else cast(Dict[str, object], general_settings_entry.param_value)
+            )
 
             # Extract email_settings from general settings if it exists
             if general_settings and "email_settings" in general_settings:
-                email_settings = general_settings["email_settings"]
+                email_settings: Final = cast(Dict[str, bool], general_settings["email_settings"])
                 # Update settings_dict with values from general_settings
                 for event_name, enabled in email_settings.items():
                     settings_dict[event_name] = enabled
@@ -64,7 +66,7 @@ async def _save_email_settings(prisma_client, settings: Dict[str, bool]):
     from litellm.proxy.proxy_server import proxy_config
 
     proxy_config.reject_config_owned_writes(
-        section_name="general_settings", changed_keys={"email_settings": settings}
+        section_name="general_settings", changed_keys={"email_settings": cast(JsonValue, settings)}
     )
     try:
         verbose_proxy_logger.debug(
@@ -77,16 +79,15 @@ async def _save_email_settings(prisma_client, settings: Dict[str, bool]):
         )
 
         # Initialize general settings dict
-        if (
-            general_settings_entry is not None
-            and general_settings_entry.param_value is not None
-        ):
-            if isinstance(general_settings_entry.param_value, str):
-                general_settings = json.loads(general_settings_entry.param_value)
-            else:
-                general_settings = dict(general_settings_entry.param_value)
-        else:
-            general_settings = {}
+        general_settings: Final = (
+            (
+                cast(Dict[str, object], json.loads(general_settings_entry.param_value))
+                if isinstance(general_settings_entry.param_value, str)
+                else cast(Dict[str, object], dict(general_settings_entry.param_value))
+            )
+            if general_settings_entry is not None and general_settings_entry.param_value is not None
+            else {}
+        )
 
         # Update email_settings in general_settings
         general_settings["email_settings"] = settings

enterprise/litellm_enterprise/integrations/custom_guardrail.py

@@ -36,7 +36,7 @@ class EnterpriseCustomGuardrailHelper:
 
         proxy_server_request = data.get("proxy_server_request", {})
 
-        request_tags = StandardLoggingPayloadSetup._get_request_tags(
+        request_tags = StandardLoggingPayloadSetup.get_request_tags(
             litellm_params=data,
             proxy_server_request=proxy_server_request,
         )

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -60,6 +60,32 @@ class _ManagedObjectRow(Protocol):
     @property
     def file_object(self) -> object: ...
 
+    @property
+    def org_id(self) -> str | None: ...
+
+    @property
+    def api_key(self) -> str | None: ...
+
+    @property
+    def team_id(self) -> str | None: ...
+
+
+class _ReadableFileContent(Protocol):
+    async def read(self) -> bytes: ...
+
+
+class _HasFileContent(Protocol):
+    @property
+    def content(self) -> bytes: ...
+
+
+async def _file_content_bytes(file_content: object) -> bytes:
+    if hasattr(file_content, "content"):
+        return cast(_HasFileContent, file_content).content
+    if hasattr(file_content, "read"):
+        return await cast(_ReadableFileContent, file_content).read()
+    return cast(bytes, file_content)
+
 
 def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
     return ManagedObjectRepository(prisma_client).table
@@ -175,17 +201,21 @@ class CheckBatchCost:
             return None
 
     async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None:
-        org_id = getattr(job, "org_id", None)
+        org_id: Final = getattr(job, "org_id", None)
         if org_id:
             return org_id
-        api_key = getattr(job, "api_key", None)
-        team_id = getattr(job, "team_id", None)
+        api_key: Final = getattr(job, "api_key", None)
+        team_id: Final = getattr(job, "team_id", None)
         if api_key:
             try:
                 key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
                     self.prisma_client
                 ).find_unique(where={"token": api_key})
-                key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None
+                key_org_id: Final = (
+                    cast(str | None, getattr(key_row, "organization_id", None))
+                    if key_row is not None
+                    else None
+                )
                 if key_org_id:
                     return key_org_id
             except Exception as e:
@@ -199,7 +229,7 @@ class CheckBatchCost:
             team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
                 where={"team_id": team_id}
             )
-            return getattr(team_row, "organization_id", None) if team_row is not None else None
+            return cast(str | None, getattr(team_row, "organization_id", None)) if team_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not resolve the team's org for batch {batch_id}: {e}")
             return None
@@ -673,16 +703,17 @@ class CheckBatchCost:
 
         from litellm.types.utils import LiteLLMBatch
 
-        file_object = job.file_object
-        if isinstance(file_object, str):
-            try:
-                file_object = json.loads(file_object)
-            except (json.JSONDecodeError, ValueError):
-                return None
-        if not isinstance(file_object, dict):
+        file_object: Final = job.file_object
+        try:
+            parsed_file_object: Final[object] = (
+                json.loads(file_object) if isinstance(file_object, str) else file_object
+            )
+        except (json.JSONDecodeError, ValueError):
+            return None
+        if not isinstance(parsed_file_object, dict):
             return None
         try:
-            return LiteLLMBatch.model_validate(file_object).input_file_id
+            return LiteLLMBatch.model_validate(parsed_file_object).input_file_id
         except Exception:
             return None
 
@@ -704,10 +735,11 @@ class CheckBatchCost:
         """
         from litellm.batches.batch_utils import (
             count_error_file_failed_requests,
-            _get_file_content_as_dictionary,
+            get_file_content_as_dictionary,
             calculate_batch_cost_and_usage,
         )
         from litellm.files.main import afile_content
+        from litellm.files.types import FileContentCallOptions
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
         from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
@@ -745,30 +777,22 @@ class CheckBatchCost:
             except (IndexError, AttributeError):
                 pass
 
-        credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
+        credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
         _file_content = await afile_content(
-            file_id=raw_output_file_id,
+            file_id=cast(str, raw_output_file_id),
             _litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
-            **credentials,
+            **cast(FileContentCallOptions, credentials),
         )
 
-        # Access content - handle both direct attribute and method call
-        if hasattr(_file_content, 'content'):
-            content_bytes = _file_content.content  # type: ignore[union-attr]
-        elif hasattr(_file_content, 'read'):
-            content_bytes = await _file_content.read()  # type: ignore[misc]
-        else:
-            content_bytes = _file_content  # type: ignore[assignment]
+        content_bytes: Final = await _file_content_bytes(_file_content)
 
-        file_content_as_dict = _get_file_content_as_dictionary(
-            content_bytes  # type: ignore[arg-type]
-        )
+        file_content_as_dict = get_file_content_as_dictionary(content_bytes)
 
         # Record output file size
         if prom_logger and content_bytes:
             try:
                 prom_logger.record_managed_file_size(
-                    size_bytes=len(content_bytes),  # type: ignore
+                    size_bytes=len(content_bytes),
                     purpose="batch",
                     file_type="output",
                     model=model_id,
@@ -805,7 +829,7 @@ class CheckBatchCost:
                 team_id=getattr(job, "team_id", None),
             )
             for _file_attr in ["output_file_id", "error_file_id"]:
-                _raw_file_id = getattr(response, _file_attr, None)
+                _raw_file_id = cast(str | None, getattr(response, _file_attr, None))
                 if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
                     try:
                         _unified_file_id = managed_files_hook.get_unified_output_file_id(

enterprise/litellm_enterprise/proxy/enterprise_routes.py

@@ -8,7 +8,7 @@ from . import ui_crud_endpoints  # side-effect: registers extra UI settings
 from .audit_logging_endpoints import router as audit_logging_router
 from .liteadmin import router as liteadmin_router
 from .management_endpoints import management_endpoints_router
-from .utils import _should_block_robots
+from .utils import should_block_robots
 
 __all__ = ["router", "ui_crud_endpoints"]
 
@@ -25,7 +25,7 @@ async def get_robots():
     Block all web crawlers from indexing the proxy server endpoints
     This is useful for ensuring that the API endpoints aren't indexed by search engines
     """
-    if _should_block_robots():
+    if should_block_robots():
         return Response(content="User-agent: *\nDisallow: /", media_type="text/plain")
     else:
         return Response(status_code=404)

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -23,6 +23,7 @@ from uuid import NAMESPACE_URL, uuid5
 import httpx
 from fastapi import HTTPException
 from pydantic import ValidationError
+from typing_extensions import ReadOnly
 
 import litellm
 from litellm import Router, verbose_logger
@@ -30,10 +31,13 @@ from litellm._internal_context import with_service_target
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.constants import MAX_FILE_LIST_LIMIT
+from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
+from openai import AsyncOpenAI
 from openai.types.file_deleted import FileDeleted
 
 from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
@@ -190,6 +194,19 @@ class _ManagedObjectTableActions(Protocol):
     async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
 
 
+class _ManagedResourceDatabase(Protocol):
+    @property
+    def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
+
+    @property
+    def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
+
+
+class _ManagedResourcePrismaClient(Protocol):
+    @property
+    def db(self) -> _ManagedResourceDatabase: ...
+
+
 class _SchedulerWithJobLookup(Protocol):
     def get_job(self, job_id: str) -> object: ...
 
@@ -199,7 +216,12 @@ class _CursorPageArgs(TypedDict, total=False):
     skip: int
 
 
-def _managed_file_table(prisma_client: PrismaClient) -> _ManagedFileTableActions:
+class _RouterFileCallKwargs(TypedDict, total=False):
+    client: ReadOnly[AsyncOpenAI | None]
+    custom_llm_provider: ReadOnly[str | None]
+
+
+def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
     return prisma_client.db.litellm_managedfiletable
 
 
@@ -213,7 +235,7 @@ def _iter_provider_file_id_pairs(
                 yield provider_file_id, row.unified_file_id
 
 
-def _managed_object_table(prisma_client: PrismaClient) -> _ManagedObjectTableActions:
+def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedObjectTableActions:
     return prisma_client.db.litellm_managedobjecttable
 
 
@@ -233,7 +255,7 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s
 _MANAGED_FILES_TARGET: Final = "managed_files"
 
 
-class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
+class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     # Class variables or attributes
     def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
         self.internal_usage_cache = internal_usage_cache
@@ -1282,7 +1304,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             target_model_names_list=target_model_names_list,
             litellm_parent_otel_span=litellm_parent_otel_span,
         )
-        response = await _PROXY_LiteLLMManagedFiles.return_unified_file_id(
+        response = await PROXY_LiteLLMManagedFiles.return_unified_file_id(
             file_objects=responses,
             create_file_request=create_file_request,
             internal_usage_cache=self.internal_usage_cache,
@@ -1294,7 +1316,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         model_mappings: Dict[str, str] = {}
 
         for file_object in responses:
-            model_file_id_mapping = file_object._hidden_params.get("model_file_id_mapping")
+            file_hidden_params = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
+                dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
+            )
+            model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
             if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
                 model_mappings.update(model_file_id_mapping)
 
@@ -1344,7 +1369,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         _, file_type = extract_file_metadata(create_file_request["file"])
 
         output_file_id = file_objects[0].id
-        model_id = file_objects[0]._hidden_params.get("model_id")
+        file_hidden_params: Final = cast(dict[str, object], getattr(file_objects[0], HIDDEN_PARAMS_ATTR))
+        model_id = file_hidden_params.get("model_id")
 
         unified_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
             file_type,
@@ -1410,11 +1436,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             if decoded_batch_id and is_litellm_executed_batch(decoded_batch_id):
                 return response
             ## Check if unified_file_id is in the response
-            unified_file_id = response._hidden_params.get("unified_file_id")  # managed file id
-            unified_batch_id = response._hidden_params.get("unified_batch_id")  # managed batch id
-            is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
-            model_id = cast(Optional[str], response._hidden_params.get("model_id"))
-            model_name = cast(Optional[str], response._hidden_params.get("model_name"))
+            response_hidden_params: Final = cast(dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR))
+            unified_file_id = response_hidden_params.get("unified_file_id")
+            unified_batch_id = response_hidden_params.get("unified_batch_id")
+            is_batch_create: Final = response_hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
+            model_id = cast(Optional[str], response_hidden_params.get("model_id"))
+            model_name = cast(Optional[str], response_hidden_params.get("model_name"))
 
             resolved_model_name = resolve_managed_output_file_model_name(
                 unified_input_file_id=unified_file_id if isinstance(unified_file_id, str) else response.input_file_id,
@@ -1459,13 +1486,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                                 _creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
                                 file_object = await litellm.afile_retrieve(
                                     file_id=provider_file_id,
-                                    **_creds,
+                                    **cast(FileRetrieveCallOptions, _creds),
                                 )
                             else:
                                 file_object = await litellm.afile_retrieve(
-                                    custom_llm_provider=model_name.split("/")[0]
-                                    if model_name and "/" in model_name
-                                    else "openai",  # type: ignore[arg-type]
+                                    custom_llm_provider=cast(
+                                        FileRetrieveProvider,
+                                        model_name.split("/")[0] if model_name and "/" in model_name else "openai",
+                                    ),
                                     file_id=provider_file_id,
                                 )
                             verbose_logger.debug(
@@ -1525,12 +1553,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
         elif isinstance(response, LiteLLMFineTuningJob):
             ## Check if unified_file_id is in the response
-            unified_file_id = response._hidden_params.get("unified_file_id")  # managed file id
-            unified_finetuning_job_id = response._hidden_params.get(
-                "unified_finetuning_job_id"
-            )  # managed finetuning job id
-            model_id = cast(Optional[str], response._hidden_params.get("model_id"))
-            model_name = cast(Optional[str], response._hidden_params.get("model_name"))
+            finetuning_response_hidden_params: Final = cast(  # cast-ok: preserve dynamic mapping behavior
+                dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
+            )
+            unified_file_id = finetuning_response_hidden_params.get("unified_file_id")
+            unified_finetuning_job_id = finetuning_response_hidden_params.get("unified_finetuning_job_id")
+            model_id = cast(Optional[str], finetuning_response_hidden_params.get("model_id"))
             original_response_id = response.id
             if (unified_file_id or unified_finetuning_job_id) and model_id:
                 response.id = self.get_unified_generic_response_id(model_id=model_id, generic_response_id=response.id)
@@ -1608,8 +1636,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
         try:
             model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
-            credentials = llm_router.get_deployment_credentials_with_provider(model_id) or {}
-            response = await litellm.afile_retrieve(file_id=model_file_id, **credentials)
+            credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id) or {}
+            response = await litellm.afile_retrieve(
+                file_id=model_file_id,
+                **cast(FileRetrieveCallOptions, credentials),
+            )
             response.id = file_id  # Replace with unified ID
             return response
         except Exception as e:
@@ -1904,7 +1935,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     else {}
                 ),
             }
-            await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
+            await llm_router.afile_delete(
+                model=model_id,
+                file_id=model_file_id,
+                **cast(_RouterFileCallKwargs, delete_data),
+            )
 
     async def afile_content(
         self,
@@ -1940,7 +1975,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
                     else:
                         data.pop("_litellm_internal_model_credentials", None)
-                    return await llm_router.afile_content(model=model_id, file_id=provider_file_id, **data)  # type: ignore
+                    return cast(
+                        HttpxBinaryResponseContent,
+                        await llm_router.afile_content(
+                            model=model_id,
+                            file_id=provider_file_id,
+                            **cast(_RouterFileCallKwargs, data),
+                        ),
+                    )
                 except Exception as e:
                     exception_dict[model_id] = str(e)
             raise Exception(
@@ -2068,3 +2110,4 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                                 verbose_logger.debug(
                                     f"Converted file {file_id} from storage backend to base64 with format {content_type}"
                                 )
+_PROXY_LiteLLMManagedFiles = PROXY_LiteLLMManagedFiles

enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py

@@ -2,11 +2,11 @@
 ## This hook is used to manage vector stores with target_model_names support
 ## It allows creating vector stores across multiple models and managing them with unified IDs
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Union, cast
 
 from fastapi import HTTPException
 
-import litellm
 from litellm import Router, verbose_logger
 from litellm._uuid import uuid
 from litellm.integrations.custom_logger import CustomLogger
@@ -38,7 +38,7 @@ else:
     PrismaClient = Any
 
 
-class _PROXY_LiteLLMManagedVectorStores(
+class PROXY_LiteLLMManagedVectorStores(
     CustomLogger, BaseManagedResource[VectorStoreCreateResponse]
 ):
     """
@@ -89,10 +89,10 @@ class _PROXY_LiteLLMManagedVectorStores(
         
         # Model ID is stored in hidden params if the response object supports it
         # For TypedDict responses, we need to check if _hidden_params was added
-        hidden_params: Dict[str, Any] = {}
+        hidden_params: Mapping[str, object] = {}
         if hasattr(resource_object, "_hidden_params"):
-            hidden_params = getattr(resource_object, "_hidden_params", {}) or {}
-        model_id = hidden_params.get("model_id", "")
+            hidden_params = cast(Mapping[str, object], getattr(resource_object, "_hidden_params", {}) or {})
+        model_id: Final = cast(str, hidden_params.get("model_id", ""))
 
         return generate_unified_id_string(
             resource_type=self.resource_type,
@@ -106,7 +106,7 @@ class _PROXY_LiteLLMManagedVectorStores(
         self,
         llm_router: Router,
         model: str,
-        request_data: Dict[str, Any],
+        request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
         litellm_parent_otel_span: Span,
     ) -> VectorStoreCreateResponse:
         """
@@ -122,10 +122,8 @@ class _PROXY_LiteLLMManagedVectorStores(
             VectorStoreCreateResponse from the provider
         """
         # Use the router to create the vector store
-        response = await llm_router.avector_store_create(
-            model=model, **request_data
-        )
-        return response
+        response: Final = await llm_router.avector_store_create(model=model, **request_data)
+        return cast(VectorStoreCreateResponse, response)
 
     # ============================================================================
     #                     VECTOR STORE CRUD OPERATIONS
@@ -464,3 +462,4 @@ class _PROXY_LiteLLMManagedVectorStores(
             parent_otel_span=parent_otel_span,
             resource_id_key="vector_store_id",
         )
+_PROXY_LiteLLMManagedVectorStores = PROXY_LiteLLMManagedVectorStores

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -22,7 +22,9 @@ from litellm._uuid import uuid
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.management.teams.authz import is_team_admin
+from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamAccess, is_team_admin, is_team_member
+from litellm.proxy.management.teams.dependencies import get_team_access
+from litellm.proxy.management.users.service import org_admin_org_ids
 from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
 from litellm.proxy.management_helpers.utils import (
@@ -118,6 +120,37 @@ async def _check_user_permission_for_project(
     return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
 
 
+async def _can_view_team_projects(
+    user_api_key_dict: UserAPIKeyAuth,
+    team_id: str | None,
+    prisma_client: PrismaClient,
+    team_access: TeamAccess,
+) -> bool:
+    if user_api_key_has_admin_view(user_api_key_dict):
+        return True
+    if not team_id or not user_api_key_dict.user_id:
+        return False
+    team_row: Final = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
+    if team_row is None:
+        return False
+    team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
+    return is_team_member(user_api_key_dict, team) or await team_access.allows(
+        user_api_key_dict, team, TEAM_OR_ORG_ADMIN
+    )
+
+
+def _visible_projects_where(team_ids: list[str], admin_org_ids: frozenset[str]) -> dict[str, object]:
+    # bounded-ok: one caller's team memberships
+    member_scope: Final[dict[str, object]] = {"team_id": {"in": team_ids}}
+    if not admin_org_ids:
+        return member_scope
+    org_scope: Final[dict[str, object]] = {
+        # bounded-ok: orgs one caller administers
+        "litellm_team_table": {"is": {"organization_id": {"in": sorted(admin_org_ids)}}}
+    }
+    return {"OR": [member_scope, org_scope]}
+
+
 async def _validate_team_exists(
     team_id: str,
     prisma_client: PrismaClient,
@@ -973,6 +1006,7 @@ async def delete_project(
 async def project_info(
     project_id: str,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    team_access: TeamAccess = Depends(get_team_access),
 ):
     """
     Get information about a specific project
@@ -1009,21 +1043,7 @@ async def project_info(
                 param="project_id",
             )
 
-        # Check if user has access to this project (admin or team member)
-        is_admin = user_api_key_has_admin_view(user_api_key_dict)
-        is_team_member = False
-
-        if project.team_id and user_api_key_dict.user_id:
-            team = await _team_table(prisma_client).find_unique(where={"team_id": project.team_id})
-            if team:
-                caller_user_id = user_api_key_dict.user_id
-                for m in team.members_with_roles or []:
-                    m_user_id = m.get("user_id") if isinstance(m, dict) else getattr(m, "user_id", None)
-                    if m_user_id == caller_user_id:
-                        is_team_member = True
-                        break
-
-        if not (is_admin or is_team_member):
+        if not await _can_view_team_projects(user_api_key_dict, project.team_id, prisma_client, team_access):
             raise HTTPException(
                 status_code=403,
                 detail={"error": "You don't have access to this project"},
@@ -1072,16 +1092,13 @@ async def list_projects(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
-            # Look up the user's team memberships via the reverse-index on
-            # LiteLLM_UserTable.teams (maintained by team_member_add alongside
-            # members_with_roles). This avoids a full scan of all team rows.
             user_record: Final = await _user_table(prisma_client).find_unique(
                 where={"user_id": user_api_key_dict.user_id},
+                include={"organization_memberships": True},
             )
-            user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
-
+            user: Final = None if user_record is None else LiteLLM_UserTable.model_validate(user_record.model_dump())
             projects = await _project_table(prisma_client).find_many(
-                where={"team_id": {"in": user_team_ids}},
+                where=_visible_projects_where(user.teams if user is not None else [], org_admin_org_ids(user)),
                 include={"litellm_budget_table": True, "object_permission": True},
             )
 

enterprise/litellm_enterprise/proxy/proxy_server.py

@@ -9,7 +9,7 @@ custom_auth_settings: Optional[CustomAuthSettings] = None
 class EnterpriseProxyConfig:
     async def load_custom_auth_settings(
         self, general_settings: dict
-    ) -> CustomAuthSettings:
+    ) -> Optional[CustomAuthSettings]:
         custom_auth_settings = general_settings.get("custom_auth_settings", None)
         if custom_auth_settings is not None:
             custom_auth_settings = CustomAuthSettings(

enterprise/litellm_enterprise/proxy/utils.py

@@ -3,7 +3,7 @@ from typing import Optional, Union
 from litellm.secret_managers.main import str_to_bool
 
 
-def _should_block_robots():
+def should_block_robots():
     """
     Returns True if the robots.txt file should block web crawlers
 
@@ -33,3 +33,4 @@ def _should_block_robots():
             )
         return True
     return False
+_should_block_robots = should_block_robots

enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py

@@ -171,7 +171,7 @@ async def list_vector_stores(
     try:
         # Get vector stores from database (source of truth)
         # Only return what's in the database to ensure consistency across instances
-        vector_stores_from_db = await VectorStoreRegistry._get_vector_stores_from_db(
+        vector_stores_from_db = await VectorStoreRegistry.get_vector_stores_from_db(
             prisma_client=prisma_client
         )
         

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.73"
+version = "0.1.74"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.73"
+version = "0.1.74"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_mcp_catalog_revision_trigger/migration.sql

@@ -0,0 +1,13 @@
+CREATE OR REPLACE FUNCTION litellm_bump_mcp_catalog_revision() RETURNS TRIGGER AS $$
+BEGIN
+  INSERT INTO "LiteLLM_Config" ("param_name", "reload_revision")
+  VALUES ('mcp_catalog', 1)
+  ON CONFLICT ("param_name") DO UPDATE SET "reload_revision" = "LiteLLM_Config"."reload_revision" + 1;
+  RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS "litellm_mcp_catalog_revision" ON "LiteLLM_MCPServerTable";
+CREATE TRIGGER "litellm_mcp_catalog_revision"
+AFTER INSERT OR UPDATE OR DELETE ON "LiteLLM_MCPServerTable"
+FOR EACH STATEMENT EXECUTE PROCEDURE litellm_bump_mcp_catalog_revision();

litellm-proxy-extras/litellm_proxy_extras/migrations/20261006000100_lens_datasets/migration.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_LensDataset" (
+    "id" TEXT NOT NULL,
+    "revision" INTEGER NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL,
+    "data" JSONB NOT NULL,
+    CONSTRAINT "LiteLLM_LensDataset_pkey" PRIMARY KEY ("id", "revision")
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261007000000_lens_review_fk_on_update_cascade/migration.sql

@@ -0,0 +1,15 @@
+-- DropForeignKey
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_LensReview_lens_id_fkey' AND conrelid = '"LiteLLM_LensReview"'::regclass) THEN
+        ALTER TABLE "LiteLLM_LensReview" DROP CONSTRAINT "LiteLLM_LensReview_lens_id_fkey";
+    END IF;
+END $$;
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_LensReview_lens_id_fkey' AND conrelid = '"LiteLLM_LensReview"'::regclass) THEN
+        ALTER TABLE "LiteLLM_LensReview" ADD CONSTRAINT "LiteLLM_LensReview_lens_id_fkey" FOREIGN KEY ("lens_id") REFERENCES "LiteLLM_Lens"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+    END IF;
+END $$;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1965,3 +1965,12 @@ model LiteLLM_LensWorker {
   token_hash String @unique
   data       Json
 }
+
+model LiteLLM_LensDataset {
+  id         String
+  revision   Int
+  created_at DateTime
+  data       Json
+
+  @@id([id, revision])
+}

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.105"
+version = "0.4.106"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.105"
+version = "0.4.106"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/traces-cache/src/cache.rs

@@ -58,9 +58,8 @@ impl SnapshotKey {
     }
 }
 
-/// How long a read result stays reusable: traces still receiving spans, or read with spend
-/// unavailable, are re-read after `LIVE_TTL`; traces quiet for `SETTLED_AFTER_MS` are kept for
-/// `SETTLED_TTL`.
+/// Native sessions can resume without a terminal record, so their reads retain `LIVE_TTL`.
+/// Other traces with known spend settle after `SETTLED_AFTER_MS` of inactivity.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum Freshness {
     Live,
@@ -69,6 +68,12 @@ pub enum Freshness {
 
 impl Freshness {
     pub fn of(rows: &[TraceSpansRow], spend_known: bool, snapshot_ms: u64) -> Self {
+        if rows
+            .iter()
+            .any(|row| matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk"))
+        {
+            return Self::Live;
+        }
         let last_end_ms = rows
             .iter()
             .map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns) / 1_000_000)
@@ -393,4 +398,18 @@ mod tests {
             expected
         );
     }
+
+    #[rstest]
+    #[case::claude_code("claude-code")]
+    #[case::claude_agent_sdk("claude-agent-sdk")]
+    fn idle_native_sessions_remain_live(#[case] framework: &str) {
+        let native = TraceSpansRow {
+            framework: framework.into(),
+            ..row("native")
+        };
+        assert_eq!(
+            Freshness::of(&[row("root"), native], true, LAST_END_MS + SETTLED_AFTER_MS),
+            Freshness::Live
+        );
+    }
 }

litellm-rust/crates/traces-cache/tests/read.rs

@@ -706,6 +706,54 @@ async fn failed_reads_are_not_cached() {
     assert_eq!(store.calls(Operation::TraceSpans), 2);
 }
 
+#[rstest]
+#[case::claude_code("claude-code")]
+#[case::claude_agent_sdk("claude-agent-sdk")]
+#[tokio::test]
+async fn resumed_native_sessions_refresh_after_live_ttl(#[case] framework: &str) {
+    let original = TraceSpansRow {
+        framework: framework.into(),
+        ..span(0)
+    };
+    let store = FakeStore::with_spans("ref", vec![original.clone()]);
+    let reader = TraceReader::new(usize::MAX);
+    let access = access();
+    let first = reader
+        .get_trace(&store, &access, "trace", "ref")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(first.spans.len(), 1);
+
+    store.state.lock().unwrap().trace_spans.insert(
+        "ref".into(),
+        vec![
+            original,
+            TraceSpansRow {
+                start_ns: now_ns(),
+                ..span(1)
+            },
+        ],
+    );
+    let cached = reader
+        .get_trace(&store, &access, "trace", "ref")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(cached.spans.len(), 1);
+    assert_eq!(store.calls(Operation::TraceSpans), 1);
+
+    tokio::time::sleep(LIVE_TTL + Duration::from_millis(200)).await;
+    let resumed = reader
+        .get_trace(&store, &access, "trace", "ref")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(resumed.spans.len(), 2);
+    assert_eq!(store.calls(Operation::TraceSpans), 2);
+    assert_eq!(first.spans.len(), 1);
+}
+
 #[rstest]
 #[tokio::test]
 async fn listed_runs_are_read_once_until_a_live_run_expires() {

litellm-rust/crates/traces-clickhouse/tests/reads.rs

@@ -600,9 +600,17 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals(
 }
 
 #[rstest]
+#[case::same_key("same-key", Some(0.25))]
+#[case::other_key("other-key", None)]
+#[case::foreign_team("foreign-team", None)]
+#[case::call_id_other_key("call-id-other-key", None)]
+#[case::call_id_foreign_team("call-id-foreign-team", None)]
+#[case::transport_only("transport-only", None)]
 #[tokio::test]
 async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads(
     #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+    #[case] id: &str,
+    #[case] expected: Option<f64>,
 ) -> TestResult {
     let fixture = migrated_database?;
     let client = &fixture.database.client;
@@ -705,20 +713,18 @@ async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_rea
         .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
         .await?;
     assert_eq!(page.data.len(), cases.len());
-    for (id, _, _, _, expected) in cases {
-        let summary = page
-            .data
-            .iter()
-            .find(|summary| summary.trace_id == id)
-            .ok_or("missing run")?;
-        let detail = reader
-            .get_trace(&store, &access, id, &summary.trace_ref)
-            .await?
-            .ok_or("missing trace")?;
-        assert_eq!(detail.summary.spend, expected, "{id}");
-        assert_eq!(summary.spend, expected, "{id}");
-        assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}");
-    }
+    let summary = page
+        .data
+        .iter()
+        .find(|summary| summary.trace_id == id)
+        .ok_or("missing run")?;
+    let detail = reader
+        .get_trace(&store, &access, id, &summary.trace_ref)
+        .await?
+        .ok_or("missing trace")?;
+    assert_eq!(detail.summary.spend, expected, "{id}");
+    assert_eq!(summary.spend, expected, "{id}");
+    assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}");
     Ok(())
 }
 

litellm-rust/crates/traces/src/normalize/format/claude_code.rs

@@ -159,10 +159,23 @@ fn exported_tool_results(attributes: &BTreeMap<String, String>) -> String {
     let Ok(body) = serde_json::from_str::<Value>(attr(attributes, "body")) else {
         return json!({"warning": "Claude's API body export is missing or truncated. Some tool results may be unavailable."}).to_string();
     };
-    let results: Vec<Value> = body.get("messages").and_then(Value::as_array)
-        .and_then(|messages| messages.last())
-        .filter(|message| message.get("role").and_then(Value::as_str) == Some("user"))
-        .and_then(|message| message.get("content").and_then(Value::as_array))
+    let message = body
+        .get("messages")
+        .and_then(Value::as_array)
+        .and_then(|messages| {
+            messages
+                .iter()
+                .rev()
+                .find(|message| message.get("role").and_then(Value::as_str) != Some("system"))
+        })
+        .filter(|message| message.get("role").and_then(Value::as_str) == Some("user"));
+    let Some(content) = message.and_then(|message| message.get("content")) else {
+        return json!({"warning": "Claude's API body export has an unexpected message shape. Some tool results may be unavailable."}).to_string();
+    };
+    if !content.is_array() && !content.is_string() {
+        return json!({"warning": "Claude's API body export has an unexpected content shape. Some tool results may be unavailable."}).to_string();
+    }
+    let results: Vec<Value> = content.as_array()
         .into_iter()
         .flatten()
         .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result"))

litellm-rust/crates/traces/src/normalize/messages.rs

@@ -422,11 +422,10 @@ pub(super) fn event_payload(events: &[(bool, Value)], output: bool) -> Option<St
     (!values.is_empty()).then(|| canonical(&encode(&values)))
 }
 
-/// The latest user message with text.
+/// The first user message with text.
 pub(super) fn preview(messages: &[Message]) -> String {
     messages
         .iter()
-        .rev()
         .find(|message| message.role == "user" && !message.content.is_empty())
         .map_or("", |message| message.content.as_str())
         .chars()
@@ -434,7 +433,7 @@ pub(super) fn preview(messages: &[Message]) -> String {
         .collect()
 }
 
-/// The latest user message when `input` is a conversation, else the input itself.
+/// The first user message when `input` is a conversation, else the input itself.
 pub(super) fn input_preview(input: &str) -> String {
     match serde_json::from_str::<Value>(input)
         .ok()
@@ -597,7 +596,7 @@ mod tests {
     use serde_json::Value;
 
     #[rstest]
-    #[case::latest_user(r#"{"messages":[{"role":"user","content":"first"},{"role":"assistant","content":"reply"},{"role":"user","content":"last"}]}"#, Some("last"))]
+    #[case::first_user(r#"{"messages":[{"role":"user","content":"first"},{"role":"assistant","content":"reply"},{"role":"user","content":"last"}]}"#, Some("first"))]
     #[case::malformed("not-json", None)]
     #[case::missing("{}", None)]
     #[case::not_messages(r#"{"messages":[{"role":"user"}]}"#, None)]

litellm-rust/crates/traces/src/otlp/logs.rs

@@ -28,16 +28,31 @@ fn text<'a>(attributes: &'a [KeyValue], key: &str) -> &'a str {
     }
 }
 
-fn message(record: LogRecord) -> Span {
+fn absent_id(id: &[u8], length: usize) -> bool {
+    id.is_empty() || (id.len() == length && id.iter().all(|byte| *byte == 0))
+}
+
+struct LogContext {
+    missing_trace: bool,
+    missing_parent: bool,
+}
+
+fn message(record: LogRecord) -> (Span, LogContext) {
     let timestamp = if record.time_unix_nano == 0 {
         record.observed_time_unix_nano
     } else {
         record.time_unix_nano
     };
+    let missing_trace = absent_id(&record.trace_id, 16);
+    let missing_parent = absent_id(&record.span_id, 8);
     let mut hash = Sha256::new();
     hash.update(b"litellm.claude.message.v1\0");
-    hash.update(&record.trace_id);
-    hash.update(&record.span_id);
+    if !missing_trace {
+        hash.update(&record.trace_id);
+    }
+    if !missing_parent {
+        hash.update(&record.span_id);
+    }
     hash.update(text(&record.attributes, "event.name"));
     let uuid = text(&record.attributes, "message.uuid");
     if uuid.is_empty() {
@@ -50,28 +65,53 @@ fn message(record: LogRecord) -> Span {
     } else {
         hash.update(uuid);
     }
+    if missing_trace {
+        hash.update(b"\0unassigned\0");
+        hash.update(text(&record.attributes, "session.id"));
+    }
+    let identity = hash.finalize();
+    let trace_id = if missing_trace {
+        let session = text(&record.attributes, "session.id");
+        if session.is_empty() {
+            identity[..16].to_vec()
+        } else {
+            span::session_trace_id(session)
+        }
+    } else {
+        record.trace_id
+    };
     let failed = match value(&record.attributes, "success") {
         Some(Value::BoolValue(success)) => !success,
         Some(Value::StringValue(success)) => success == "false",
         _ => false,
     };
-    Span {
-        trace_id: record.trace_id,
-        span_id: hash.finalize()[..8].to_vec(),
-        parent_span_id: record.span_id,
-        name: format!("claude_code.{}", text(&record.attributes, "event.name")),
-        kind: 1,
-        start_time_unix_nano: timestamp,
-        end_time_unix_nano: timestamp,
-        status: (text(&record.attributes, "event.name") == "tool_result" && failed).then(|| {
-            Status {
-                code: 2,
-                message: text(&record.attributes, "error").to_owned(),
-            }
-        }),
-        attributes: record.attributes,
-        ..Span::default()
-    }
+    (
+        Span {
+            trace_id,
+            span_id: identity[..8].to_vec(),
+            parent_span_id: if missing_trace || missing_parent {
+                Vec::new()
+            } else {
+                record.span_id
+            },
+            name: format!("claude_code.{}", text(&record.attributes, "event.name")),
+            kind: 1,
+            start_time_unix_nano: timestamp,
+            end_time_unix_nano: timestamp,
+            status: (text(&record.attributes, "event.name") == "tool_result" && failed).then(
+                || Status {
+                    code: 2,
+                    message: text(&record.attributes, "error").to_owned(),
+                },
+            ),
+            attributes: record.attributes,
+            ..Span::default()
+        },
+        LogContext {
+            missing_trace,
+            missing_parent,
+        },
+    )
 }
 
 pub(super) fn flatten(
@@ -79,6 +119,7 @@ pub(super) fn flatten(
     limits: DecodeLimits,
 ) -> Result<Vec<DecodedSpan>, Error> {
     let mut count = 0usize;
+    let mut contexts = Vec::new();
     let mut resources = Vec::new();
     for resource in request.resource_logs {
         let mut scopes = Vec::new();
@@ -111,7 +152,11 @@ pub(super) fn flatten(
                                     text(&record.attributes, "query_source"),
                                 )))
                 })
-                .map(message)
+                .map(|record| {
+                    let (span, context) = message(record);
+                    contexts.push(context);
+                    span
+                })
                 .collect();
             scopes.push(ScopeSpans {
                 scope: scope.scope,
@@ -125,10 +170,23 @@ pub(super) fn flatten(
             schema_url: resource.schema_url,
         });
     }
-    span::flatten(
+    let (mut spans, mut budget) = span::flatten_with_budget(
         ExportTraceServiceRequest {
             resource_spans: resources,
         },
         limits,
-    )
+    )?;
+    budget.consume(contexts.len() * size_of::<LogContext>())?;
+    for (span, context) in spans.iter_mut().zip(contexts) {
+        if context.missing_trace {
+            span.attributes.remove("lens.original_trace_id");
+        }
+        if context.missing_trace || context.missing_parent {
+            let key = "lens.capture.warning";
+            let warning = "This native log has incomplete trace context. Its content is retained, but its execution parent is unconfirmed.";
+            budget.consume(key.len() + warning.len() + 96)?;
+            span.attributes.insert(key.to_owned(), warning.to_owned());
+        }
+    }
+    Ok(spans)
 }

litellm-rust/crates/traces/src/otlp/span.rs

@@ -20,12 +20,19 @@ pub(super) fn flatten(
     request: ExportTraceServiceRequest,
     limits: DecodeLimits,
 ) -> Result<Vec<DecodedSpan>, Error> {
+    flatten_with_budget(request, limits).map(|(spans, _)| spans)
+}
+
+pub(super) fn flatten_with_budget(
+    request: ExportTraceServiceRequest,
+    limits: DecodeLimits,
+) -> Result<(Vec<DecodedSpan>, Budget), Error> {
     let mut budget = Budget::new(limits);
     let mut spans = Vec::new();
     for resource in request.resource_spans {
         append_resource(resource, &mut budget, &mut spans)?;
     }
-    Ok(spans)
+    Ok((spans, budget))
 }
 
 fn append_resource(
@@ -124,6 +131,10 @@ fn hex_bytes(bytes: &[u8]) -> String {
     bytes.iter().map(|byte| format!("{byte:02x}")).collect()
 }
 
+pub(super) fn session_trace_id(session: &str) -> Vec<u8> {
+    Sha256::digest(format!("litellm.claude.session.v1\0{session}"))[..16].to_vec()
+}
+
 fn decoded_span(
     span: Span,
     resource_attributes: &Shared<BTreeMap<String, String>>,
@@ -145,8 +156,7 @@ fn decoded_span(
             .get("session.id")
             .filter(|value| !value.is_empty())
     {
-        let trace_id =
-            hex_bytes(&Sha256::digest(format!("litellm.claude.session.v1\0{session}"))[..16]);
+        let trace_id = hex_bytes(&session_trace_id(session));
         let actor = span_attributes.get("agent_id").unwrap_or(session).clone();
         budget.consume(original_trace_id.len() + actor.len() + 256)?;
         span_attributes.insert("lens.original_trace_id".to_owned(), original_trace_id);

litellm-rust/crates/traces/src/ui.rs

@@ -85,6 +85,32 @@ struct RawMessage {
     kwargs: Option<Box<RawMessage>>,
 }
 
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct AssistantSummary {
+    content: Option<String>,
+    tool_names: Vec<String>,
+}
+
+impl AssistantSummary {
+    fn into_ui(self) -> UiMessage {
+        let calls: Vec<UiToolCall> = self
+            .tool_names
+            .into_iter()
+            .map(|name| UiToolCall {
+                name,
+                arguments: "{}".to_owned(),
+            })
+            .collect();
+        UiMessage {
+            role: ChatRole::Assistant,
+            content: self.content.unwrap_or_default(),
+            name: None,
+            tool_calls: (!calls.is_empty()).then_some(calls),
+        }
+    }
+}
+
 #[derive(Deserialize)]
 struct ContentBlock {
     #[serde(rename = "type", default)]
@@ -211,6 +237,19 @@ fn messages(parsed: &Value) -> Option<Vec<UiMessage>> {
     Some(unwrapped.into_iter().map(RawMessage::into_ui).collect())
 }
 
+fn assistant_summaries(parsed: &Value) -> Option<Vec<UiMessage>> {
+    let summaries = Vec::<AssistantSummary>::deserialize(parsed).ok()?;
+    if summaries.is_empty() {
+        return None;
+    }
+    Some(
+        summaries
+            .into_iter()
+            .map(AssistantSummary::into_ui)
+            .collect(),
+    )
+}
+
 pub fn to_ui_content(raw: &str) -> UiContent {
     let text = || UiContent::Text {
         text: raw.to_owned(),
@@ -223,7 +262,7 @@ pub fn to_ui_content(raw: &str) -> UiContent {
         Ok(parsed @ (Value::Array(_) | Value::Object(_))) => parsed,
         _ => return text(),
     };
-    if let Some(messages) = messages(&parsed) {
+    if let Some(messages) = messages(&parsed).or_else(|| assistant_summaries(&parsed)) {
         return UiContent::Messages { messages };
     }
     match parsed {
@@ -356,6 +395,33 @@ mod tests {
         );
     }
 
+    #[rstest]
+    fn assistant_summaries_become_assistant_messages() {
+        let raw = json!([
+            {"content": "`/etc/hosts` has 11 lines.", "tool_names": []},
+            {"content": null, "tool_names": ["terminal"]},
+        ]);
+        assert_eq!(
+            to_ui_content(&raw.to_string()),
+            UiContent::Messages {
+                messages: vec![
+                    message("assistant", "`/etc/hosts` has 11 lines."),
+                    UiMessage {
+                        tool_calls: Some(vec![call("terminal", "{}")]),
+                        ..message("assistant", "")
+                    },
+                ]
+            }
+        );
+    }
+
+    #[rstest]
+    #[case::extra_field(r#"[{"content": "x", "tool_names": [], "score": 1}]"#)]
+    #[case::missing_tool_names(r#"[{"content": "x"}]"#)]
+    fn near_summaries_stay_text(#[case] raw: &str) {
+        assert!(matches!(to_ui_content(raw), UiContent::Text { .. }));
+    }
+
     #[rstest]
     fn plain_objects_become_fields_in_key_order() {
         let raw = r#"{"zeta": "plain", "alpha": {"nested": [1, 2]}, "count": 3, "missing": null}"#;

litellm-rust/crates/traces/tests/normalization_formats.rs

@@ -502,6 +502,25 @@ fn existing_formats_win_over_new_formats(span: Span, #[case] kind: &str) {
     assert_eq!(decoded.normalized.observation_type, ObservationType::Llm);
 }
 
+#[rstest]
+fn input_preview_is_the_first_user_message(span: Span) {
+    let conversation = json!([
+        {"role": "system", "content": "sys"},
+        {"role": "user", "content": "initial question"},
+        {"role": "assistant", "content": "answer"},
+        {"role": "user", "content": "follow up"},
+    ])
+    .to_string();
+    let decoded = decode(
+        span,
+        "custom",
+        &[("gen_ai.input.messages", &conversation)],
+        vec![],
+    )
+    .unwrap();
+    assert_eq!(decoded.normalized.input_preview, "initial question");
+}
+
 #[rstest]
 #[case::messages(&[("gen_ai.input.messages", r#"[{"role":"user","content":"modern"}]"#)], "modern")]
 #[case::indexed(&[("gen_ai.prompt.0.role", "user"), ("gen_ai.prompt.0.content", "indexed")], "indexed")]

litellm-rust/crates/traces/tests/normalize.rs

@@ -82,7 +82,7 @@ fn assert_invariants(span: &DecodedSpan) {
     }
     assert!(normalized.input_preview.chars().count() <= 240);
     if let Ok(Value::Array(messages)) = serde_json::from_str(&normalized.input) {
-        let user = messages.iter().rev().find_map(|message| {
+        let user = messages.iter().find_map(|message| {
             (message.get("role")?.as_str()? == "user")
                 .then(|| {
                     message

litellm-rust/crates/traces/tests/otlp.rs

@@ -1673,7 +1673,7 @@ fn session_capture_joins_native_logs_and_traces_across_turns_without_changing_sp
 
 #[rstest]
 #[case::short_trace(vec![1;15], vec![2;8], 1)]
-#[case::zero_parent(vec![1;16], vec![0;8], 1)]
+#[case::short_parent(vec![1;16], vec![2;7], 1)]
 #[case::timestamp(vec![1;16], vec![2;8], i64::MAX as u64 + 1)]
 fn native_logs_reject_invalid_context(
     #[case] trace: Vec<u8>,
@@ -1692,6 +1692,106 @@ fn native_logs_reject_invalid_context(
     ));
 }
 
+#[rstest]
+#[case::json_empty(true, false, true, true)]
+#[case::protobuf_empty(false, false, true, true)]
+#[case::json_zero(true, true, true, true)]
+#[case::protobuf_zero(false, true, true, true)]
+#[case::trace_only(true, false, true, false)]
+#[case::parent_only(false, false, false, true)]
+fn contextless_native_logs_preserve_the_entire_batch(
+    #[case] json: bool,
+    #[case] zero: bool,
+    #[case] missing_trace: bool,
+    #[case] missing_parent: bool,
+) {
+    use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value::Value};
+    use prost::Message;
+    let mut request = log_request("repl_main_thread");
+    let valid = request.resource_logs[0].scope_logs[0].log_records[0].clone();
+    let mut uncorrelated = valid.clone();
+    if missing_trace {
+        uncorrelated.trace_id = if zero { vec![0; 16] } else { Vec::new() };
+    }
+    if missing_parent {
+        uncorrelated.span_id = if zero { vec![0; 8] } else { Vec::new() };
+    }
+    let mut standalone = uncorrelated.clone();
+    standalone
+        .attributes
+        .retain(|attribute| attribute.key != "session.id");
+    request.resource_logs[0].scope_logs[0].log_records = vec![valid, uncorrelated, standalone];
+    request.resource_logs[0].resource = Some(opentelemetry_proto::tonic::resource::v1::Resource {
+        attributes: vec![KeyValue {
+            key: "lens.session.capture".into(),
+            value: Some(AnyValue {
+                value: Some(Value::StringValue("true".into())),
+            }),
+            ..Default::default()
+        }],
+        ..Default::default()
+    });
+    let bytes = if json {
+        serde_json::to_vec(&request).unwrap()
+    } else {
+        request.encode_to_vec()
+    };
+    let media = json.then_some("application/json");
+    let limits = litellm_traces::DecodeLimits {
+        attributes: 6,
+        ..Default::default()
+    };
+    let spans = litellm_traces::decode_otlp_logs_with_limits(&bytes, media, limits).unwrap();
+    let replayed = litellm_traces::decode_otlp_logs_with_limits(&bytes, media, limits).unwrap();
+    assert_eq!(spans.len(), 3);
+    assert_eq!(
+        serde_json::to_value(&spans).unwrap(),
+        serde_json::to_value(&replayed).unwrap()
+    );
+    assert_eq!(spans[0].trace_id, spans[1].trace_id);
+    assert_ne!(spans[1].trace_id, spans[2].trace_id);
+    assert_ne!(spans[0].span_id, spans[1].span_id);
+    if missing_trace {
+        assert_ne!(spans[1].span_id, spans[2].span_id);
+        assert!(!spans[1].attributes.contains_key("lens.original_trace_id"));
+    }
+    assert_eq!(spans[0].parent_span_id, "02".repeat(8));
+    assert!(spans[1].parent_span_id.is_empty());
+    assert!(spans[2].parent_span_id.is_empty());
+    assert!(!spans[0].attributes.contains_key("lens.capture.warning"));
+    assert!(spans[1].attributes["lens.capture.warning"].contains("unconfirmed"));
+    assert!(spans[2].attributes["lens.capture.warning"].contains("unconfirmed"));
+    assert_eq!(spans[0].normalized.output, spans[1].normalized.output);
+    assert_eq!(spans[0].normalized.output, spans[2].normalized.output);
+    assert_eq!(spans[1].normalized.observation_type, ObservationType::Chain);
+}
+
+#[rstest]
+#[case::session(true)]
+#[case::standalone(false)]
+fn absent_native_log_context_has_encoding_independent_identity(#[case] session: bool) {
+    use prost::Message;
+    let mut request = log_request("repl_main_thread");
+    let record = &mut request.resource_logs[0].scope_logs[0].log_records[0];
+    record.trace_id.clear();
+    record.span_id.clear();
+    if !session {
+        record.attributes.retain(|entry| entry.key != "session.id");
+    }
+    let omitted = litellm_traces::decode_otlp_logs(
+        &serde_json::to_vec(&request).unwrap(),
+        Some("application/json"),
+    )
+    .unwrap();
+    let record = &mut request.resource_logs[0].scope_logs[0].log_records[0];
+    record.trace_id = vec![0; 16];
+    record.span_id = vec![0; 8];
+    let zeroed = litellm_traces::decode_otlp_logs(&request.encode_to_vec(), None).unwrap();
+    assert_eq!(omitted[0].trace_id, zeroed[0].trace_id);
+    assert_eq!(omitted[0].span_id, zeroed[0].span_id);
+    assert_eq!(omitted[0].normalized.output, zeroed[0].normalized.output);
+}
+
 #[rstest]
 #[case::nodes(litellm_traces::DecodeLimits { nodes: 4, ..Default::default() })]
 #[case::depth(litellm_traces::DecodeLimits { depth: 2, ..Default::default() })]
@@ -1778,6 +1878,18 @@ fn interactive_claude_exports_join_replies_with_native_child_execution_context()
 #[rstest]
 #[case::tool_result("tool_result", "", false)]
 #[case::complete_body("api_request_body", r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":[{"type":"text","text":"exit 3 output"},{"type":"image","source":{"data":"PRIVATE_IMAGE"}}]}]}],"system":"PRIVATE_SYSTEM"}"#, false)]
+#[case::headless_body("api_request_body", r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":"exit 3 output"}]},{"role":"system","content":"PRIVATE_SYSTEM"}]}"#, false)]
+#[case::missing_messages("api_request_body", r#"{}"#, true)]
+#[case::wrong_content(
+    "api_request_body",
+    r#"{"messages":[{"role":"user","content":{}}]}"#,
+    true
+)]
+#[case::unexpected_last_message(
+    "api_request_body",
+    r#"{"messages":[{"role":"assistant","content":"unexpected"}]}"#,
+    true
+)]
 #[case::truncated_body("api_request_body", "{truncated", true)]
 fn native_tool_logs_supply_arguments_and_results_without_fake_calls(
     #[case] event: &str,
@@ -1849,3 +1961,39 @@ fn interactive_claude_body_export_retains_failed_command_stdout() {
         .unwrap();
     assert_eq!(failed["content"], "Exit code 3\nRAW-EXPECTED");
 }
+
+#[rstest]
+#[case::new_prompt(serde_json::json!({"role":"user", "content":"Continue"}))]
+#[case::new_blocks(serde_json::json!({"role":"user", "content":[{"type":"text", "text":"Continue"}]}))]
+fn native_body_exports_do_not_replay_old_tool_results(#[case] final_message: serde_json::Value) {
+    use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value::Value};
+    let mut request = log_request("repl_main_thread");
+    let body = serde_json::json!({"messages": [
+        {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "old-call", "content": "OLD_RESULT"}]},
+        {"role": "assistant", "content": "Done"}, final_message,
+        {"role": "system", "content": "PRIVATE_SYSTEM"}
+    ]}).to_string();
+    request.resource_logs[0].scope_logs[0].log_records[0].attributes = [
+        ("event.name", "api_request_body"),
+        ("query_source", "repl_main_thread"),
+        ("body", body.as_str()),
+    ]
+    .into_iter()
+    .map(|(key, value)| KeyValue {
+        key: key.into(),
+        value: Some(AnyValue {
+            value: Some(Value::StringValue(value.into())),
+        }),
+        ..Default::default()
+    })
+    .collect();
+    let spans = litellm_traces::decode_otlp_logs(
+        &serde_json::to_vec(&request).unwrap(),
+        Some("application/json"),
+    )
+    .unwrap();
+    assert_eq!(
+        serde_json::from_str::<serde_json::Value>(&spans[0].normalized.output).unwrap(),
+        serde_json::json!({"tool_results":[]})
+    );
+}

litellm/__init__.py

@@ -53,10 +53,10 @@ from litellm.types.integrations.pointfive import PointFiveInitParams
 from litellm.types.integrations.zerobus import ZerobusInitParams
 from litellm._logging import (
     set_verbose,
-    _turn_on_debug,
+    turn_on_debug,
     verbose_logger,
     json_logs,
-    _turn_on_json,
+    turn_on_json,
     log_level,
 )
 import re
@@ -108,10 +108,10 @@ litellm_mode = os.getenv("LITELLM_MODE", "DEV")  # "PRODUCTION", "DEV"
 
 ####################################################
 if set_verbose:
-    _turn_on_debug()
+    turn_on_debug()
 ####################################################
 ### Callbacks /Logging / Success / Failure Handlers #####
-CALLBACK_TYPES = Union[str, Callable, "CustomLogger"]  # CustomLogger is lazy-loaded
+CALLBACK_TYPES = Union[str, Callable[..., object], "CustomLogger"]  # CustomLogger is lazy-loaded
 input_callback: List[CALLBACK_TYPES] = []
 success_callback: List[CALLBACK_TYPES] = []
 failure_callback: List[CALLBACK_TYPES] = []
@@ -177,9 +177,9 @@ _custom_logger_compatible_callbacks_literal = Literal[
 ]
 cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
 logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
-_known_custom_logger_compatible_callbacks: List = list(get_args(_custom_logger_compatible_callbacks_literal))
+_known_custom_logger_compatible_callbacks: List[str] = list(get_args(_custom_logger_compatible_callbacks_literal))
 callbacks: List[
-    Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"]  # CustomLogger is lazy-loaded
+    Union[Callable[..., object], str, "CustomLogger"]  # CustomLogger is lazy-loaded
 ] = []
 callback_settings: Dict[str, Dict[str, Any]] = {}
 initialized_langfuse_clients: int = 0
@@ -194,13 +194,13 @@ datadog_use_v1: Optional[bool] = False  # if you want to use v1 datadog logged p
 gcs_pub_sub_use_v1: Optional[bool] = False  # if you want to use v1 gcs pubsub logged payload
 generic_api_use_v1: Optional[bool] = False  # if you want to use v1 generic api logged payload
 argilla_transformation_object: Optional[Dict[str, Any]] = None
-_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = (  # CustomLogger is lazy-loaded
+_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = (  # CustomLogger is lazy-loaded
     []
 )  # internal variable - async custom callbacks are routed here.
-_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = (  # CustomLogger is lazy-loaded
+_async_success_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = (  # CustomLogger is lazy-loaded
     []
 )  # internal variable - async custom callbacks are routed here.
-_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = (  # CustomLogger is lazy-loaded
+_async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = (  # CustomLogger is lazy-loaded
     []
 )  # internal variable - async custom callbacks are routed here.
 pre_call_rules: List[Callable] = []
@@ -1404,6 +1404,7 @@ from .exceptions import (
     BadGatewayError,
     OpenAIError,
     ContextWindowExceededError,
+    PaymentRequiredError as PaymentRequiredError,
     ContentPolicyViolationError,
     BudgetExceededError,
     APIError,
@@ -1486,6 +1487,7 @@ from .realtime_api.main import (
     arealtime_calls,
 )
 from .responses.main import _aresponses_websocket
+
 from .fine_tuning.main import *
 from .files.main import *
 from .vector_store_files.main import (
@@ -1527,6 +1529,9 @@ from . import rag
 ### CUSTOM LLMs ###
 from .types.llms.custom_llm import CustomLLMItem
 
+_turn_on_debug = turn_on_debug
+_turn_on_json = turn_on_json
+
 custom_provider_map: List[CustomLLMItem] = []
 _custom_providers: List[str] = []  # internal helper util, used to track names of custom providers
 disable_hf_tokenizer_download: Optional[bool] = (
@@ -2240,7 +2245,9 @@ if TYPE_CHECKING:
     register_model: Callable[..., None]
     encode: Callable[..., list]
     decode: Callable[..., str]
+    calculate_retry_after: Callable[..., float]
     _calculate_retry_after: Callable[..., float]
+    should_retry: Callable[..., bool]
     _should_retry: Callable[..., bool]
     get_supported_openai_params: Callable[..., Optional[list]]
     get_api_base: Callable[..., Optional[str]]
@@ -2336,9 +2343,9 @@ def __getattr__(name: str) -> Any:
         _async_client_cleanup_registered = True
 
     # Use cached registry from _lazy_imports instead of importing tuples every time
-    from ._lazy_imports import _get_lazy_import_registry
+    from ._lazy_imports import get_lazy_import_registry
 
-    registry: Final = _get_lazy_import_registry()
+    registry: Final = get_lazy_import_registry()
 
     # Check if name is in registry and call the cached handler function
     if name in registry:

litellm/_lazy_imports.py

@@ -91,12 +91,15 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
 # They're separate from the main lazy import system because they have specific use cases
 
 
-def _get_default_encoding() -> "Tokenizer":
+def get_default_encoding() -> "Tokenizer":
     from litellm.rust_bridge.tokenizer import get_encoding
 
     return get_encoding("cl100k_base")
 
 
+_get_default_encoding = get_default_encoding
+
+
 # Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
 _get_modified_max_tokens_func: "Callable[..., int | None] | None" = None
 
@@ -126,7 +129,7 @@ _token_counter_new_func: "Callable[..., int] | None" = None
 _messages_reach_token_count_func: "Callable[..., bool] | None" = None
 
 
-def _get_token_counter_new() -> "Callable[..., int]":
+def get_token_counter_new() -> "Callable[..., int]":
     """
     Lazily load and cache the token_counter function (aliased as token_counter_new).
 
@@ -146,8 +149,11 @@ def _get_token_counter_new() -> "Callable[..., int]":
     return _token_counter_new_func
 
 
-def _get_messages_reach_token_count() -> "Callable[..., bool]":
-    """Lazily load ``messages_reach_token_count`` for the same reason as ``_get_token_counter_new``."""
+_get_token_counter_new = get_token_counter_new
+
+
+def get_messages_reach_token_count() -> "Callable[..., bool]":
+    """Lazily load ``messages_reach_token_count`` for the same reason as ``get_token_counter_new``."""
     global _messages_reach_token_count_func
     if _messages_reach_token_count_func is None:
         from litellm.litellm_core_utils.token_counter import (
@@ -158,6 +164,9 @@ def _get_messages_reach_token_count() -> "Callable[..., bool]":
     return _messages_reach_token_count_func
 
 
+_get_messages_reach_token_count = get_messages_reach_token_count
+
+
 # ============================================================================
 # MAIN LAZY IMPORT SYSTEM
 # ============================================================================
@@ -168,7 +177,7 @@ def _get_messages_reach_token_count() -> "Callable[..., bool]":
 _LAZY_IMPORT_REGISTRY: dict[str, Callable[[str], object]] | None = None
 
 
-def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
+def get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
     """
     Build the registry that maps attribute names to their handler functions.
 
@@ -217,6 +226,9 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
     return _LAZY_IMPORT_REGISTRY
 
 
+_get_lazy_import_registry = get_lazy_import_registry
+
+
 class _AttributeView(TypedDict):
     """Holds one module attribute so the lazily fetched value is read back as ``object``."""
 

litellm/_lazy_imports_registry.py

@@ -48,7 +48,9 @@ UTILS_NAMES: Final = (
     "register_model",
     "encode",
     "decode",
+    "calculate_retry_after",
     "_calculate_retry_after",
+    "should_retry",
     "_should_retry",
     "get_supported_openai_params",
     "get_api_base",
@@ -384,15 +386,18 @@ UTILS_MODULE_NAMES: Final = (
     "_get_response_headers",
     "get_llm_provider",
     "_is_non_openai_azure_model",
+    "is_non_openai_azure_model",
     "get_supported_openai_params",
     "LiteLLMResponseObjectHandler",
     "_handle_invalid_parallel_tool_calls",
+    "handle_invalid_parallel_tool_calls",
     "convert_to_model_response_object",
     "convert_to_streaming_response",
     "convert_to_streaming_response_async",
     "get_api_base",
     "ResponseMetadata",
     "_parse_content_for_reasoning",
+    "parse_content_for_reasoning",
     "LiteLLMLoggingObject",
     "redact_message_input_output_from_logging",
     "CustomStreamWrapper",
@@ -415,8 +420,10 @@ UTILS_MODULE_NAMES: Final = (
     "delete_nested_value",
     "is_nested_path",
     "_get_base_model_from_litellm_call_metadata",
+    "get_base_model_from_litellm_call_metadata",
     "get_litellm_params",
     "_ensure_extra_body_is_safe",
+    "ensure_extra_body_is_safe",
     "get_formatted_prompt",
     "get_response_headers",
     "update_response_metadata",
@@ -475,8 +482,10 @@ _UTILS_IMPORT_MAP: Final = {
     "register_model": (".utils", "register_model"),
     "encode": (".utils", "encode"),
     "decode": (".utils", "decode"),
-    "_calculate_retry_after": (".utils", "_calculate_retry_after"),
-    "_should_retry": (".utils", "_should_retry"),
+    "calculate_retry_after": (".utils", "calculate_retry_after"),
+    "_calculate_retry_after": (".utils", "calculate_retry_after"),
+    "should_retry": (".utils", "should_retry"),
+    "_should_retry": (".utils", "should_retry"),
     "get_supported_openai_params": (".utils", "get_supported_openai_params"),
     "get_api_base": (".utils", "get_api_base"),
     "get_first_chars_messages": (".utils", "get_first_chars_messages"),
@@ -1320,7 +1329,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
     ),
     "_is_non_openai_azure_model": (
         "litellm.litellm_core_utils.get_llm_provider_logic",
-        "_is_non_openai_azure_model",
+        "is_non_openai_azure_model",
+    ),
+    "is_non_openai_azure_model": (
+        "litellm.litellm_core_utils.get_llm_provider_logic",
+        "is_non_openai_azure_model",
     ),
     "get_supported_openai_params": (
         "litellm.litellm_core_utils.get_supported_openai_params",
@@ -1332,7 +1345,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
     ),
     "_handle_invalid_parallel_tool_calls": (
         "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response",
-        "_handle_invalid_parallel_tool_calls",
+        "handle_invalid_parallel_tool_calls",
+    ),
+    "handle_invalid_parallel_tool_calls": (
+        "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response",
+        "handle_invalid_parallel_tool_calls",
     ),
     "convert_to_model_response_object": (
         "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response",
@@ -1356,7 +1373,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
     ),
     "_parse_content_for_reasoning": (
         "litellm.litellm_core_utils.prompt_templates.common_utils",
-        "_parse_content_for_reasoning",
+        "parse_content_for_reasoning",
+    ),
+    "parse_content_for_reasoning": (
+        "litellm.litellm_core_utils.prompt_templates.common_utils",
+        "parse_content_for_reasoning",
     ),
     "LiteLLMLoggingObject": (
         "litellm.litellm_core_utils.redact_messages",
@@ -1429,7 +1450,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
     ),
     "_get_base_model_from_litellm_call_metadata": (
         "litellm.litellm_core_utils.get_litellm_params",
-        "_get_base_model_from_litellm_call_metadata",
+        "get_base_model_from_litellm_call_metadata",
+    ),
+    "get_base_model_from_litellm_call_metadata": (
+        "litellm.litellm_core_utils.get_litellm_params",
+        "get_base_model_from_litellm_call_metadata",
     ),
     "get_litellm_params": (
         "litellm.litellm_core_utils.get_litellm_params",
@@ -1437,7 +1462,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
     ),
     "_ensure_extra_body_is_safe": (
         "litellm.litellm_core_utils.llm_request_utils",
-        "_ensure_extra_body_is_safe",
+        "ensure_extra_body_is_safe",
+    ),
+    "ensure_extra_body_is_safe": (
+        "litellm.litellm_core_utils.llm_request_utils",
+        "ensure_extra_body_is_safe",
     ),
     "get_formatted_prompt": (
         "litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt",

litellm/_logging.py

@@ -23,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int
 from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.secret_redaction import (
-    _python_redact_string,
-    _python_redact_structured_value,
+    python_redact_string,
+    python_redact_structured_value,
     redact_internal_details,
-    redact_string,
+)
+from litellm.litellm_core_utils.secret_redaction import (
+    redact_string as redact_secret_string,
 )
 from litellm.rust_bridge import diagnostics
 
@@ -52,7 +54,7 @@ def _sanitize_correlation_id(value: str) -> str:
     pass through credential redaction.
     """
     stripped: Final = "".join(ch for ch in value if ch.isprintable())
-    return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
+    return redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
 
 
 def set_session_id(session_id: str) -> "contextvars.Token[str]":
@@ -71,10 +73,13 @@ if set_verbose is True:
 _ENABLE_SECRET_REDACTION: Final = os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
 
 
-def _redact_string(value: str) -> str:
+def redact_string(value: str) -> str:
     if not _ENABLE_SECRET_REDACTION:
         return value
-    return redact_string(value)
+    return redact_secret_string(value)
+
+
+_redact_string = redact_string
 
 
 _REDACTED_RECORD_ATTR: Final = "litellm_redacted"
@@ -114,7 +119,7 @@ def redact_secrets(value: str) -> str:
     """
     if not _ENABLE_SECRET_REDACTION:
         return value
-    return _redact_string(value)
+    return redact_string(value)
 
 
 def redact_internal_details_from_client_message(value: str) -> str:
@@ -165,7 +170,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED"
 def _hides_a_credential(value: str) -> bool:
     """Whether *value* only looks clean until it is percent-decoded."""
     decoded: Final = unquote(value)
-    return _python_redact_string(decoded) != decoded
+    return python_redact_string(decoded) != decoded
 
 
 def _drop_encoded_credential(scrubbed: str) -> str:
@@ -193,10 +198,10 @@ def _scrub_access_arg(value: str) -> str:
     pattern and would then be logged raw.
     """
     if len(value) <= _MAX_SCRUBBED_ACCESS_ARG:
-        return _drop_encoded_credential(_python_redact_string(value))
+        return _drop_encoded_credential(python_redact_string(value))
     head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG]
     kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head
-    scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept))
+    scrubbed: Final = _drop_encoded_credential(python_redact_string(kept))
     return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..."
 
 
@@ -383,14 +388,14 @@ def _python_process_diagnostic(
 ) -> tuple[str, str | None, str | None, tuple[str, ...], bool]:
     def process_text(text: str) -> str:
         collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text
-        scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed
+        scrubbed: Final = python_redact_string(collapsed) if redact else collapsed
         return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed
 
     processed_message: Final = process_text(message)
     processed_exception: Final = process_text(exception) if exception is not None else None
-    processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack
+    processed_stack: Final = python_redact_string(stack) if redact and stack is not None else stack
     processed_leaves: Final = tuple(
-        _python_redact_structured_value(key, text) if redact else text for key, text in leaves
+        python_redact_structured_value(key, text) if redact else text for key, text in leaves
     )
     changed: Final = (
         processed_message != message
@@ -472,7 +477,7 @@ def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit:
         record.stack_info = processed_stack  # rebind-ok: the Filter interface mutates the record
     processed_values: Final = iter(processed_leaves[: len(extra_leaves)])
     for key, original, prepared in extras:
-        replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
+        replacement = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
         if not _scrubbing_changed_nothing(replacement, original):
             setattr(record, key, replacement)
     raw_color_changed: Final = (
@@ -489,12 +494,12 @@ def _redact_json_record(value: object) -> object:
     leaves: Final = tuple(_string_leaves(None, prepared))
     candidate: Final = diagnostics.run(
         lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3],
-        lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves),
+        lambda: tuple(python_redact_structured_value(key, text) for key, text in leaves),
     )
     replacements: Final = (
         candidate
         if len(candidate) == len(leaves)
-        else tuple(_python_redact_structured_value(key, text) for key, text in leaves)
+        else tuple(python_redact_structured_value(key, text) for key, text in leaves)
     )
     return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements)))
 
@@ -791,7 +796,7 @@ class CorrelationPlainFormatter(logging.Formatter):
 
     def format(self, record: logging.LogRecord) -> str:
         rendered: Final = super().format(record)
-        formatted: Final = rendered if _is_redacted(record) else _redact_string(rendered)
+        formatted: Final = rendered if _is_redacted(record) else redact_string(rendered)
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:
@@ -1050,7 +1055,7 @@ def _get_uvicorn_json_log_config():
     return log_config
 
 
-def _turn_on_json():
+def turn_on_json() -> None:
     """
     Turn on JSON logging
 
@@ -1064,12 +1069,18 @@ def _turn_on_json():
     _setup_json_exception_handlers(JsonFormatter())
 
 
-def _turn_on_debug():
+_turn_on_json = turn_on_json
+
+
+def turn_on_debug() -> None:
     verbose_logger.setLevel(level=logging.DEBUG)  # set package log to debug
     verbose_router_logger.setLevel(level=logging.DEBUG)  # set router logs to debug
     verbose_proxy_logger.setLevel(level=logging.DEBUG)  # set proxy logs to debug
 
 
+_turn_on_debug = turn_on_debug
+
+
 def _disable_debugging():
     """Disable the package, router, and proxy verbose loggers."""
     verbose_logger.disabled = True
@@ -1093,8 +1104,11 @@ def print_verbose(print_statement):
         pass
 
 
-def _is_debugging_on() -> bool:
+def is_debugging_on() -> bool:
     """
     Returns True if debugging is on
     """
     return verbose_logger.isEnabledFor(logging.DEBUG) or set_verbose is True
+
+
+_is_debugging_on = is_debugging_on

litellm/_redis.py

@@ -26,7 +26,7 @@ from litellm._redis_credential_provider import (
     AzureADCredentialProvider,
     ElastiCacheIAMCredentialProvider,
     GCPIAMCredentialProvider,
-    _generate_gcp_iam_access_token,
+    generate_gcp_iam_access_token,
 )
 from litellm.constants import (
     REDIS_CLUSTER_HEALTH_CHECK_INTERVAL,
@@ -37,6 +37,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 
 from ._logging import verbose_logger
 
+_generate_gcp_iam_access_token = generate_gcp_iam_access_token
+
 AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default"
 
 _AWS_IAM_KWARG_NAMES: Final = (
@@ -341,7 +343,7 @@ def create_gcp_iam_redis_connect_func(
 
         self._parser.on_connect(self)
 
-        auth_args: Final = (_generate_gcp_iam_access_token(service_account),)
+        auth_args: Final = (generate_gcp_iam_access_token(service_account),)
         self.send_command("AUTH", *auth_args, check_health=False)
 
         try:

litellm/_redis_credential_provider.py

@@ -38,7 +38,7 @@ class AzureCredential(Protocol):
     def get_token(self, *scopes: str) -> AzureAccessToken: ...
 
 
-def _generate_gcp_iam_access_token(service_account: str) -> str:
+def generate_gcp_iam_access_token(service_account: str) -> str:
     """
     Generate GCP IAM access token for Redis authentication.
 
@@ -65,6 +65,9 @@ def _generate_gcp_iam_access_token(service_account: str) -> str:
     return str(response.access_token)
 
 
+_generate_gcp_iam_access_token = generate_gcp_iam_access_token
+
+
 def _get_cached_gcp_iam_token(service_account: str) -> str:
     """
     Return a cached GCP IAM token, refreshing only when expired.
@@ -93,7 +96,7 @@ def _get_cached_gcp_iam_token(service_account: str) -> str:
             if time.monotonic() < expiry:
                 return token
 
-        token = _generate_gcp_iam_access_token(service_account)
+        token = generate_gcp_iam_access_token(service_account)
         _token_cache[service_account] = (
             token,
             time.monotonic() + _GCP_IAM_TOKEN_TTL_SECONDS,

litellm/batches/batch_utils.py

@@ -3,7 +3,7 @@ from collections.abc import Iterable, Iterator, Mapping
 from dataclasses import dataclass
 from dataclasses import replace as dataclasses_replace
 from enum import Enum
-from typing import Any, Final, Literal
+from typing import Any, Final, Literal, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -70,7 +70,7 @@ def batch_cost_is_final(batch: Batch) -> bool:
 
 
 async def calculate_batch_cost_and_usage(
-    file_content_dictionary: list[dict],
+    file_content_dictionary: list[dict[str, object]],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
@@ -96,11 +96,11 @@ async def calculate_batch_cost_and_usage(
     )
 
 
-async def _handle_completed_batch(
+async def handle_completed_batch(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
-    litellm_params: dict | None = None,
+    litellm_params: dict[str, object] | None = None,
     model_info: ModelInfo | None = None,
 ) -> BatchCostUsageResult:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
@@ -162,6 +162,9 @@ async def _handle_completed_batch(
     )
 
 
+_handle_completed_batch = handle_completed_batch
+
+
 class _LineOutcome(Enum):
     """A batch output line that yielded no billable stats."""
 
@@ -183,7 +186,7 @@ class _BatchOutputLineStats:
 
 
 def _classify_output_line_stats(
-    entries: Iterable[dict],
+    entries: Iterable[dict[str, object]],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
@@ -295,7 +298,7 @@ def _output_line_cost(
 
 
 def _aggregate_batch_cost_usage_models(
-    entries: Iterable[dict],
+    entries: Iterable[dict[str, object]],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
@@ -347,7 +350,7 @@ def _aggregate_batch_cost_usage_models(
 
 
 def calculate_vertex_ai_batch_cost_and_usage(
-    vertex_ai_batch_responses: Iterable[dict],
+    vertex_ai_batch_responses: Iterable[dict[str, object]],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
 ) -> BatchCostUsageResult:
@@ -436,7 +439,7 @@ def _provider_output_file_id(output_file_id: str) -> str:
 async def _fetch_batch_managed_file_content(
     file_id: str,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
-    litellm_params: dict | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> bytes:
     """
     Fetch a batch's output or error file and return its raw JSONL bytes.
@@ -448,17 +451,18 @@ async def _fetch_batch_managed_file_content(
                        Required for Azure and other providers that need authentication
     """
     from litellm.files.main import afile_content
+    from litellm.files.types import FileContentCallOptions, FileContentRequestKwargs
 
-    # Build kwargs for afile_content with credentials from litellm_params
-    file_content_kwargs: Final = {
-        "file_id": _provider_output_file_id(file_id),
+    provider_output_file_id: Final = _provider_output_file_id(file_id)
+    credentials: Final = extract_file_access_credentials(litellm_params)
+    file_content_kwargs: Final[FileContentRequestKwargs] = {
+        "file_id": provider_output_file_id,
         "custom_llm_provider": custom_llm_provider,
+        **cast(  # cast-ok: preserve dynamic provider credentials without validation
+            FileContentCallOptions, credentials
+        ),
     }
 
-    # Extract and add credentials for file access
-    credentials: Final = _extract_file_access_credentials(litellm_params)
-    file_content_kwargs.update(credentials)
-
     _file_content: Final = await afile_content(**file_content_kwargs)
     return _file_content.content
 
@@ -466,7 +470,7 @@ async def _fetch_batch_managed_file_content(
 async def _fetch_batch_output_file_content(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
-    litellm_params: dict | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> bytes:
     """
     Fetch the batch output file and return its raw JSONL bytes
@@ -488,7 +492,7 @@ async def _fetch_batch_output_file_content(
 async def count_error_file_failed_requests(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
-    litellm_params: dict | None,
+    litellm_params: Mapping[str, object] | None,
 ) -> int:
     """Count failed requests reported only in the batch's separate error file.
 
@@ -506,10 +510,12 @@ async def count_error_file_failed_requests(
     except Exception as e:  # noqa: BLE001  # a failed/missing error file must not abort cost tracking for the batch
         verbose_logger.debug("Failed to fetch batch error file %s: %s", batch.error_file_id, e)
         return 0
-    return sum(1 for _ in _iter_batch_input_lines(error_file_content))
+    return sum(1 for _ in iter_batch_input_lines(error_file_content))
 
 
-def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
+def extract_file_access_credentials(
+    litellm_params: Mapping[str, object] | None,
+) -> dict[str, object]:
     """
     Extract credentials from litellm_params for file access operations.
 
@@ -522,7 +528,7 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
     Returns:
         Dictionary containing only the credentials needed for file access
     """
-    credentials: Final = {}
+    credentials: Final[dict[str, object]] = {}
 
     if litellm_params:
         # List of credential keys that should be passed to file operations
@@ -555,7 +561,10 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
     return credentials
 
 
-def _get_file_content_as_dictionary(file_content: bytes) -> list[dict]:
+_extract_file_access_credentials = extract_file_access_credentials
+
+
+def get_file_content_as_dictionary(file_content: bytes) -> list[dict[str, object]]:
     """
     Get the file content as a list of dictionaries from JSON Lines format,
     skipping malformed lines
@@ -563,7 +572,10 @@ def _get_file_content_as_dictionary(file_content: bytes) -> list[dict]:
     return list(_iter_batch_output_entries(file_content))
 
 
-def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]:
+_get_file_content_as_dictionary = get_file_content_as_dictionary
+
+
+def iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]:
     """
     Yield non-empty JSONL lines (unparsed) one at a time, so a caller can parse
     each row in its own try/except and a single malformed line cannot abort the
@@ -581,27 +593,30 @@ def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]:
             yield line
 
 
-def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict]:
+_iter_batch_input_lines = iter_batch_input_lines
+
+
+def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict[str, object]]:
     """
     Yield parsed batch output JSONL entries one at a time without materializing
     the whole file as a list, so peak memory stays bounded. A malformed or
     non-object line is skipped with a warning so one bad line never aborts the
     whole batch's cost accounting.
     """
-    for line in _iter_batch_input_lines(file_content):
+    for line in iter_batch_input_lines(file_content):
         entry = _parse_batch_output_line(line)
         if entry is not None:
             yield entry
 
 
-def _parse_batch_output_line(line: bytes) -> dict | None:
+def _parse_batch_output_line(line: bytes) -> dict[str, object] | None:
     try:
         parsed: Final[object] = json.loads(line)
     except ValueError as e:
         verbose_logger.warning("skipping malformed batch output line: %s", str(e))
         return None
     if isinstance(parsed, dict):
-        return parsed
+        return cast("dict[str, object]", parsed)  # cast-ok: JSON object keys are strings by definition
     verbose_logger.warning("skipping non-object batch output line of type %s", type(parsed).__name__)
     return None
 
@@ -611,24 +626,29 @@ def _parse_batch_output_line(line: bytes) -> dict | None:
 _BATCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN: Final = 4
 
 
-def _estimate_batch_entry_tokens(raw_line: bytes) -> int:
+def estimate_batch_entry_tokens(raw_line: bytes) -> int:
     """Conservative token estimate for a batch row the token counter cannot measure
     (or that cannot be parsed). Keeps the batch token total non-zero so a crafted
     row cannot evade the TPM limit, without hard-rejecting a legitimate batch."""
     return max(1, len(raw_line) // _BATCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN)
 
 
-def _count_entry_tokens(
-    entry: dict,
+_estimate_batch_entry_tokens = estimate_batch_entry_tokens
+
+
+def count_entry_tokens(
+    entry: Mapping[str, object],
     model_name: str | None = None,
 ) -> int:
     """Token-count a single batch input entry's body (chat / text / embedding)."""
-    body: Final = entry.get("body", {}) or {}
-    model: Final = body.get("model", model_name or "")
+    body: Final = cast(  # cast-ok: batch payload bodies come from provider JSON
+        Mapping[str, object], entry.get("body", {}) or {}
+    )
+    model: Final = cast(str, body.get("model", model_name or ""))  # cast-ok: provider batch model names are strings
 
     messages: Final = body.get("messages")
     if messages:
-        return token_counter(model=model, messages=messages)
+        return token_counter(model=model, messages=cast(list[dict[str, object]], messages))
 
     prompt: Final = body.get("prompt")
     if prompt:
@@ -641,6 +661,9 @@ def _count_entry_tokens(
     return 0
 
 
+_count_entry_tokens = count_entry_tokens
+
+
 def _count_prompt_or_input_tokens(model: str, value: object) -> int:
     """Token-count a ``prompt`` / ``input`` field that the OpenAI batch
     schema allows in four shapes:
@@ -707,8 +730,8 @@ def _get_batch_job_usage_from_response_body(
     from litellm.responses.utils import ResponseAPILoggingUtils
 
     _usage_dict: Final = response_body.get("usage", None) or {}
-    if ResponseAPILoggingUtils._is_response_api_usage(_usage_dict):
-        return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_usage_dict)
+    if ResponseAPILoggingUtils.is_response_api_usage(_usage_dict):
+        return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_usage_dict)
     usage: Final[Usage] = Usage(**_usage_dict)
     if custom_llm_provider == "xai":
         from litellm.llms.xai.chat.transformation import XAIChatConfig
@@ -737,9 +760,11 @@ def _get_response_from_batch_job_output_file(
         return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {}
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput", None) or {}
-    _response: Final[dict] = batch_job_output_file.get("response", None) or {}
+    _response: Final = cast(  # cast-ok: batch output response comes from provider JSON
+        Mapping[str, object], batch_job_output_file.get("response", None) or {}
+    )
     _response_body: Final = _response.get("body", None) or {}
-    return _response_body
+    return cast(Mapping[str, object], _response_body)  # cast-ok: batch response body comes from provider JSON
 
 
 def _batch_response_was_successful(
@@ -756,5 +781,7 @@ def _batch_response_was_successful(
         return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     if custom_llm_provider == "bedrock":
         return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
-    _response: Final[dict] = batch_job_output_file.get("response", None) or {}
+    _response: Final[dict[str, object]] = cast(  # cast-ok: batch response bodies come from provider JSON
+        dict[str, object], batch_job_output_file.get("response", None) or {}
+    )
     return _response.get("status_code", None) == 200

litellm/caching/caching.py

@@ -16,7 +16,7 @@ import traceback
 from collections.abc import Generator, Mapping
 from contextlib import contextmanager
 from enum import Enum
-from typing import Any, Final, Literal
+from typing import Any, Final, Literal, cast
 
 from pydantic import BaseModel
 
@@ -363,12 +363,12 @@ class Cache:
         cache_key = ""
         # verbose_logger.debug("\nGetting Cache key. Kwargs: %s", kwargs)
 
-        preset_cache_key: Final = self._get_preset_cache_key_from_kwargs(**kwargs)
+        preset_cache_key: Final = self.get_preset_cache_key_from_kwargs(**kwargs)
         if preset_cache_key is not None:
             verbose_logger.debug("\nReturning preset cache key: %s", preset_cache_key)
             return preset_cache_key
 
-        combined_kwargs: Final = ModelParamHelper._get_all_llm_api_params()
+        combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params()
         is_semantic_cache: Final = self._is_semantic_cache()
         scope_excluded_params: Final = self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS if is_semantic_cache else frozenset()
         for param in kwargs:
@@ -460,7 +460,7 @@ class Cache:
             or litellm_params.get("file_name")
         )
 
-    def _get_preset_cache_key_from_kwargs(self, **kwargs) -> str | None:
+    def get_preset_cache_key_from_kwargs(self, **kwargs: object) -> str | None:
         """
         Get the preset cache key from kwargs["litellm_params"]
 
@@ -469,11 +469,17 @@ class Cache:
         1. optional params like max_tokens, get transformed for bedrock -> max_new_tokens
         2. avoid doing duplicate / repeated work
         """
-        if kwargs:
-            if "litellm_params" in kwargs:
-                return kwargs["litellm_params"].get("preset_cache_key", None)
+        if "litellm_params" in kwargs:
+            litellm_params: Final = cast(  # cast-ok: cache kwargs retain dynamic caller values
+                Mapping[str, object], kwargs["litellm_params"]
+            )
+            return cast(  # cast-ok: preserve dynamically supplied cache keys
+                str | None, litellm_params.get("preset_cache_key", None)
+            )
         return None
 
+    _get_preset_cache_key_from_kwargs = get_preset_cache_key_from_kwargs
+
     def _set_preset_cache_key_in_kwargs(self, preset_cache_key: str, **kwargs) -> None:
         """
         Set the calculated cache key in kwargs
@@ -539,11 +545,11 @@ class Cache:
             }
             time.sleep(CACHED_STREAMING_CHUNK_DELAY)
 
-    def _get_cache_logic(
+    def get_cache_logic(
         self,
         cached_result: object | None,
         max_age: float | None,
-    ):
+    ) -> object | None:
         """
         Common get cache logic across sync + async implementations
         """
@@ -572,6 +578,8 @@ class Cache:
             return cached_response
         return cached_result
 
+    _get_cache_logic = get_cache_logic
+
     @staticmethod
     def _get_safe_cache_lookup_kwargs(kwargs: Mapping[str, object]) -> dict[str, object]:
         cache_lookup_kwargs: Final[dict[str, object]] = {}
@@ -628,7 +636,7 @@ class Cache:
                         original_kwargs=kwargs,
                         cache_lookup_kwargs=cache_lookup_kwargs,
                     )
-                    return self._get_cache_logic(cached_result=cached_result, max_age=max_age)
+                    return self.get_cache_logic(cached_result=cached_result, max_age=max_age)
         except Exception:
             print_verbose(f"An exception occurred: {traceback.format_exc()}")
             return None
@@ -658,7 +666,7 @@ class Cache:
                         cached_result = await dynamic_cache_object.async_get_cache(cache_key, **kwargs)
                     else:
                         cached_result = await self.cache.async_get_cache(cache_key, **kwargs)
-                    return self._get_cache_logic(cached_result=cached_result, max_age=max_age)
+                    return self.get_cache_logic(cached_result=cached_result, max_age=max_age)
         except Exception:
             print_verbose(f"An exception occurred: {traceback.format_exc()}")
             return None
@@ -957,7 +965,7 @@ class Cache:
         if hasattr(self.cache, "disconnect"):
             await self.cache.disconnect()
 
-    def _supports_async(self) -> bool:
+    def supports_async(self) -> bool:
         """
         Internal method to check if the cache type supports async get/set operations
 
@@ -966,6 +974,8 @@ class Cache:
         """
         return True
 
+    _supports_async = supports_async
+
 
 def enable_cache(
     type: LiteLLMCacheType | None = LiteLLMCacheType.LOCAL,

litellm/caching/caching_handler.py

@@ -29,11 +29,12 @@ from litellm._logging import print_verbose, verbose_logger
 from litellm.caching import InMemoryCache
 from litellm.caching.caching import S3Cache, response_cache_phase
 from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS
+from litellm.litellm_core_utils.hidden_params import get_hidden_params
 from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
     update_response_metadata,
 )
 from litellm.litellm_core_utils.logging_utils import (
-    _assemble_complete_response_from_streaming_chunks,
+    assemble_complete_response_from_streaming_chunks,
 )
 from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding
 from litellm.types.integrations.custom_logger import converted_stream_requested
@@ -66,7 +67,7 @@ _StreamResultT = TypeVar("_StreamResultT")
 
 
 from litellm.litellm_core_utils.core_helpers import (
-    _get_parent_otel_span_from_kwargs,
+    get_parent_otel_span_from_kwargs,
 )
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 
@@ -212,6 +213,15 @@ def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None:
     return request_kwargs.get("cache_key", None)
 
 
+def _set_cached_hidden_param(response: object, key: str, value: object) -> None:
+    if isinstance(response, TextCompletionResponse):
+        setattr(response.hidden_params, key, value)
+        return
+    hidden_params: Final = get_hidden_params(response)
+    if hidden_params is not None:
+        hidden_params[key] = value
+
+
 class _CachedEmbeddingRecord(LiteLLMBaseModel):
     model_config = ConfigDict(frozen=True)
 
@@ -246,14 +256,14 @@ def _current_format_embedding_entry(entry: object) -> CachedEmbedding | None:
 class LLMCachingHandler:
     def __init__(
         self,
-        original_function: Callable,
+        original_function: Callable[..., object],
         request_kwargs: dict[str, object],
         start_time: datetime.datetime,
     ):
         from litellm.caching import DualCache, RedisCache
 
-        self.async_streaming_chunks: list[ModelResponse] = []
-        self.sync_streaming_chunks: list[ModelResponse] = []
+        self.async_streaming_chunks: list[object] = []
+        self.sync_streaming_chunks: list[object] = []
         self.request_kwargs = _drop_logging_obj_from_kwargs(request_kwargs)
         self.preset_cache_key: str | None = None
         self.original_function = original_function
@@ -266,10 +276,10 @@ class LLMCachingHandler:
         else:
             self.dual_cache = None
 
-    async def _async_get_cache(
+    async def async_get_cache(
         self,
         model: str,
-        original_function: Callable,
+        original_function: Callable[..., object],
         logging_obj: LiteLLMLoggingObj,
         start_time: datetime.datetime,
         call_type: str,
@@ -305,7 +315,7 @@ class LLMCachingHandler:
             args = args or ()
             final_embedding_cached_response: EmbeddingResponse | None = None
             embedding_all_elements_cache_hit: bool = False
-            cached_result: Any | None = None
+            cached_result: object | None = None
             kwargs = kwargs.copy()
             #########################################################
             # Init cache timing metrics
@@ -313,7 +323,7 @@ class LLMCachingHandler:
             cache_check_start_time: Final = time.perf_counter()
             cache_check_end_time: float | None = None
             #########################################################
-            parent_otel_span: Final = _get_parent_otel_span_from_kwargs(kwargs)
+            parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
             kwargs["parent_otel_span"] = parent_otel_span
 
             if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
@@ -371,8 +381,7 @@ class LLMCachingHandler:
                         or self.request_kwargs.get("cache_key")
                         or litellm.cache.get_cache_key(**self.request_kwargs)
                     )
-                    if hasattr(cached_result, "_hidden_params"):
-                        cached_result._hidden_params["cache_key"] = cache_key
+                    _set_cached_hidden_param(cached_result, "cache_key", cache_key)
                     return CachingHandlerResponse(cached_result=cached_result)
                 elif (
                     call_type == CallTypes.aembedding.value
@@ -406,10 +415,12 @@ class LLMCachingHandler:
         # Caching disabled - return None to indicate no caching attempted
         return None
 
-    def _sync_get_cache(
+    _async_get_cache = async_get_cache
+
+    def sync_get_cache(
         self,
         model: str,
-        original_function: Callable,
+        original_function: Callable[..., object],
         logging_obj: LiteLLMLoggingObj,
         start_time: datetime.datetime,
         call_type: str,
@@ -491,11 +502,12 @@ class LLMCachingHandler:
                         or self.request_kwargs.get("cache_key")
                         or litellm.cache.get_cache_key(**self.request_kwargs)
                     )
-                    if hasattr(cached_result, "_hidden_params"):
-                        cached_result._hidden_params["cache_key"] = cache_key
+                    _set_cached_hidden_param(cached_result, "cache_key", cache_key)
                     return CachingHandlerResponse(cached_result=cached_result)
         return CachingHandlerResponse(cached_result=cached_result)
 
+    _sync_get_cache = sync_get_cache
+
     def handle_kwargs_input_list_or_str(self, kwargs: dict[str, object]) -> list[str]:
         """
         Handles the input of kwargs['input'] being a list or a string
@@ -570,7 +582,7 @@ class LLMCachingHandler:
                 model=model_name,
                 data=[None] * len(kwargs_input_as_list),
             )
-            final_embedding_cached_response._hidden_params["cache_hit"] = True
+            final_embedding_cached_response.hidden_params["cache_hit"] = True
 
             prompt_tokens = 0
             aggregated_details: dict | None = None
@@ -715,7 +727,7 @@ class LLMCachingHandler:
         except Exception:
             return None
 
-    def _combine_cached_embedding_response_with_api_result(
+    def combine_cached_embedding_response_with_api_result(
         self,
         _caching_handler_response: CachingHandlerResponse,
         embedding_response: EmbeddingResponse,
@@ -745,6 +757,7 @@ class LLMCachingHandler:
             if cached.usage is not None and embedding_response.usage is not None
             else cached.usage
         )
+        cached_hidden_params: Final = cached.hidden_params
         merged: Final = EmbeddingResponse(
             model=cached.model,
             data=[
@@ -755,7 +768,7 @@ class LLMCachingHandler:
             ],
             usage=merged_usage,
             hidden_params={
-                **cached._hidden_params,
+                **cached_hidden_params,
                 "cache_hit": True,
             },
             _response_headers=cached._response_headers,
@@ -763,6 +776,8 @@ class LLMCachingHandler:
         merged._response_ms = (end_time - start_time).total_seconds() * 1000
         return merged
 
+    _combine_cached_embedding_response_with_api_result = combine_cached_embedding_response_with_api_result
+
     def _async_log_cache_hit_on_callbacks(
         self,
         logging_obj: LiteLLMLoggingObj,
@@ -834,7 +849,7 @@ class LLMCachingHandler:
         if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
             new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
         self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
-        cached_result: Any | None = None
+        cached_result: object | None = None
         if call_type == CallTypes.aembedding.value:
             if isinstance(new_kwargs["input"], str):
                 new_kwargs["input"] = [new_kwargs["input"]]
@@ -861,7 +876,7 @@ class LLMCachingHandler:
             request_kwargs: Final = new_kwargs.copy()
             request_cache_key: Final = _request_cache_key(request_kwargs)
             request_kwargs.pop("cache_key", None)
-            if litellm.cache._supports_async() is True:
+            if litellm.cache.supports_async() is True:
                 ## check if dual cache is supported ##
                 self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
                 with response_cache_phase("get"):
@@ -1023,12 +1038,7 @@ class LLMCachingHandler:
                 )
 
                 response_obj: Final = ResponsesAPIResponse(**cached_result)
-                if (
-                    hasattr(response_obj, "_hidden_params")
-                    and response_obj._hidden_params is not None
-                    and isinstance(response_obj._hidden_params, dict)
-                ):
-                    response_obj._hidden_params["cache_hit"] = True
+                _set_cached_hidden_param(response_obj, "cache_hit", True)
 
                 if _stream_replay_requested(kwargs):
                     cached_result = CachedResponsesAPIStreamingIterator(
@@ -1040,12 +1050,7 @@ class LLMCachingHandler:
                 else:
                     cached_result = response_obj
 
-        if (
-            hasattr(cached_result, "_hidden_params")
-            and cached_result._hidden_params is not None
-            and isinstance(cached_result._hidden_params, dict)
-        ):
-            cached_result._hidden_params["cache_hit"] = True
+        _set_cached_hidden_param(cached_result, "cache_hit", True)
 
         #########################################################
         # Add final timing metrics to the cached result
@@ -1111,7 +1116,7 @@ class LLMCachingHandler:
             None
         """
         from litellm.litellm_core_utils.core_helpers import (
-            _get_parent_otel_span_from_kwargs,
+            get_parent_otel_span_from_kwargs,
         )
 
         if litellm.cache is None:
@@ -1128,10 +1133,10 @@ class LLMCachingHandler:
                 args,
             )
         )
-        parent_otel_span: Final = _get_parent_otel_span_from_kwargs(new_kwargs)
+        parent_otel_span: Final = get_parent_otel_span_from_kwargs(new_kwargs)
         new_kwargs["parent_otel_span"] = parent_otel_span
         # [OPTIONAL] ADD TO CACHE
-        if self._should_store_result_in_cache(original_function=original_function, kwargs=new_kwargs):
+        if self.should_store_result_in_cache(original_function=original_function, kwargs=new_kwargs):
             if (
                 isinstance(result, litellm.ModelResponse)
                 or isinstance(result, litellm.EmbeddingResponse)
@@ -1183,13 +1188,13 @@ class LLMCachingHandler:
             )
         )
 
-        if self._should_store_result_in_cache(original_function=self.original_function, kwargs=new_kwargs):
+        if self.should_store_result_in_cache(original_function=self.original_function, kwargs=new_kwargs):
             with response_cache_phase("set"):
                 litellm.cache.add_cache(result, **new_kwargs)
 
         return
 
-    def _should_store_result_in_cache(self, original_function: Callable, kwargs: dict[str, Any]) -> bool:
+    def should_store_result_in_cache(self, original_function: Callable[..., object], kwargs: dict[str, Any]) -> bool:
         """
         Helper function to determine if the result should be stored in the cache.
 
@@ -1200,6 +1205,8 @@ class LLMCachingHandler:
             kwargs.get("cache", {}).get("no-store", False) is not True
         )
 
+    _should_store_result_in_cache = should_store_result_in_cache
+
     def wrap_streaming_result_for_cache(
         self, result: _StreamResultT, call_type: str
     ) -> "_StreamResultT | AnthropicMessagesStreamCacheWriter":
@@ -1208,7 +1215,7 @@ class LLMCachingHandler:
             CallTypes.aanthropic_messages.value,
         ):
             return result
-        if litellm.cache is None or not self._should_store_result_in_cache(
+        if litellm.cache is None or not self.should_store_result_in_cache(
             original_function=self.original_function, kwargs=self.request_kwargs
         ):
             return result
@@ -1240,7 +1247,7 @@ class LLMCachingHandler:
         covering_call_types: Final = ("aresponses", "responses") if call_type == "aresponses" else (call_type,)
         return any(name in litellm.cache.supported_call_types for name in covering_call_types)
 
-    async def _add_streaming_response_to_cache(self, processed_chunk: ModelResponse):
+    async def add_streaming_response_to_cache(self, processed_chunk: ModelResponse) -> None:
         """
         Internal method to add the streaming response to the cache
 
@@ -1251,7 +1258,7 @@ class LLMCachingHandler:
         """
 
         complete_streaming_response: Final[ModelResponse | TextCompletionResponse | None] = (
-            _assemble_complete_response_from_streaming_chunks(
+            assemble_complete_response_from_streaming_chunks(
                 result=processed_chunk,
                 start_time=self.start_time,
                 end_time=datetime.datetime.now(),
@@ -1268,12 +1275,14 @@ class LLMCachingHandler:
             kwargs=self.request_kwargs,
         )
 
-    def _sync_add_streaming_response_to_cache(self, processed_chunk: ModelResponse):
+    _add_streaming_response_to_cache = add_streaming_response_to_cache
+
+    def sync_add_streaming_response_to_cache(self, processed_chunk: ModelResponse) -> None:
         """
         Sync internal method to add the streaming response to the cache
         """
         complete_streaming_response: Final[ModelResponse | TextCompletionResponse | None] = (
-            _assemble_complete_response_from_streaming_chunks(
+            assemble_complete_response_from_streaming_chunks(
                 result=processed_chunk,
                 start_time=self.start_time,
                 end_time=datetime.datetime.now(),
@@ -1290,6 +1299,8 @@ class LLMCachingHandler:
             kwargs=self.request_kwargs,
         )
 
+    _sync_add_streaming_response_to_cache = sync_add_streaming_response_to_cache
+
     def _update_litellm_logging_obj_environment(
         self,
         logging_obj: LiteLLMLoggingObj,
@@ -1329,7 +1340,7 @@ class LLMCachingHandler:
 
         if litellm.cache is not None:
             litellm_params["preset_cache_key"] = (
-                self.preset_cache_key or litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
+                self.preset_cache_key or litellm.cache.get_preset_cache_key_from_kwargs(**kwargs)
             )
         else:
             litellm_params["preset_cache_key"] = None

litellm/caching/redis_cache.py

@@ -38,7 +38,7 @@ from litellm.constants import (
     REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
     REDIS_TIMEOUT_LOG_INTERVAL,
 )
-from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
+from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.types.caching import (
     RedisPipelineIncrementOperation,
@@ -1157,7 +1157,7 @@ class RedisCache(BaseCache):
                     error=e,
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                     call_type="async_set_cache",
                     caller=_get_call_stack_info(),
                 )
@@ -1192,7 +1192,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
             return result
@@ -1208,7 +1208,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
             log_redis_failure(
@@ -1276,7 +1276,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
             return
@@ -1293,7 +1293,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
 
@@ -1384,7 +1384,7 @@ class RedisCache(BaseCache):
                     error=e,
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                     call_type="async_set_cache_sadd",
                     caller=_get_call_stack_info(),
                 )
@@ -1410,7 +1410,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
         except Exception as e:
@@ -1425,7 +1425,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
             # NON blocking - notify users Redis is throwing an exception
@@ -2036,7 +2036,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
             return results
@@ -2053,7 +2053,7 @@ class RedisCache(BaseCache):
                     caller=_get_call_stack_info(),
                     start_time=start_time,
                     end_time=end_time,
-                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                    parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
                 )
             )
             log_redis_failure(

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -24,8 +24,10 @@ from pydantic import BaseModel
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     responses_reasoning_items_from_thinking_blocks,
+    with_prompt_cache_breakpoint,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.bridges.completion_transformation import (
@@ -244,7 +246,7 @@ def tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _Cha
     name: Final = item.get("name") or ("custom_tool" if is_custom else "")
     function_chunk: Final = ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments)
     tool_call_dict: Final = _ChatToolCallDict(
-        id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(item.get("id"), item.get("call_id")),
+        id=LiteLLMCompletionResponsesConfig.tool_call_id_from_responses_item(item.get("id"), item.get("call_id")),
         type="function",
         function=function_chunk,
         index=index,
@@ -525,7 +527,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             elif key == "previous_response_id":
                 responses_api_request["previous_response_id"] = value
             elif key == "reasoning_effort":
-                responses_api_request["reasoning"] = self._map_reasoning_effort(value)
+                responses_api_request["reasoning"] = self.map_reasoning_effort(value)
             elif key == "web_search_options":
                 self._add_web_search_tool(responses_api_request, value)
 
@@ -983,7 +985,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         setattr(
             model_response,
             "usage",
-            ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_response.usage),
+            ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(raw_response.usage),
         )
 
         model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
@@ -992,20 +994,25 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         # Preserve hidden params from the ResponsesAPIResponse, especially the headers
         # which contain important provider information like x-request-id
-        raw_response_hidden_params: Final = getattr(raw_response, "_hidden_params", {})
+        raw_response_hidden_params: Final = get_hidden_params(raw_response) or {}
         if raw_response_hidden_params:
-            if not hasattr(model_response, "_hidden_params") or model_response._hidden_params is None:
-                model_response._hidden_params = {}
+            model_response_hidden_params: Final = get_or_create_hidden_params(model_response)
             # Merge the raw_response hidden params with model_response hidden params
             # Preserve existing keys in model_response but add/override with raw_response params
             for key, value in raw_response_hidden_params.items():
-                if key == "additional_headers" and key in model_response._hidden_params:
-                    # Merge additional_headers to preserve both sets
-                    existing_additional_headers = model_response._hidden_params.get("additional_headers", {})
-                    merged_headers = {**value, **existing_additional_headers}
-                    model_response._hidden_params[key] = merged_headers
+                if key == "additional_headers" and key in model_response_hidden_params:
+                    existing_additional_headers = model_response_hidden_params.get("additional_headers", {})
+                    merged_headers = {
+                        **cast(  # cast-ok: preserve mapping operations on dynamic response metadata
+                            "dict[str, object]", value
+                        ),
+                        **cast(  # cast-ok: preserve mapping operations on dynamic response metadata
+                            "dict[str, object]", existing_additional_headers
+                        ),
+                    }
+                    model_response_hidden_params[key] = merged_headers
                 else:
-                    model_response._hidden_params[key] = value
+                    model_response_hidden_params[key] = value
 
         return model_response
 
@@ -1089,16 +1096,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     # Handle multimodal content
                     original_type = item.get("type")
                     if original_type == "text":
-                        converted = self._convert_content_str_to_input_text(item.get("text", ""), role)
+                        converted = with_prompt_cache_breakpoint(
+                            self._convert_content_str_to_input_text(item.get("text", ""), role),
+                            item.get("prompt_cache_breakpoint"),
+                        )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
                     elif original_type == "image_url":
                         # Map to responses API image format
-                        converted = cast(
-                            dict,
-                            self._convert_content_to_responses_format_image(
-                                cast(ChatCompletionImageObject, item), role
+                        converted = with_prompt_cache_breakpoint(
+                            cast(
+                                dict,
+                                self._convert_content_to_responses_format_image(
+                                    cast(ChatCompletionImageObject, item), role
+                                ),
                             ),
+                            item.get("prompt_cache_breakpoint"),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
@@ -1110,8 +1123,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   image -> %s", converted)
                         elif item_type == "file":
-                            converted = _input_file_from_file_value(
-                                cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
+                            converted = with_prompt_cache_breakpoint(
+                                _input_file_from_file_value(
+                                    cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
+                                ),
+                                item.get("prompt_cache_breakpoint"),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1206,7 +1222,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         return optional_params
 
-    def _map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
+    def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
         # If dict is passed, convert it directly to Reasoning object
         if isinstance(reasoning_effort, dict):
             return Reasoning(
@@ -1225,6 +1241,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             else Reasoning(effort=reasoning_effort)
         )
 
+    _map_reasoning_effort = map_reasoning_effort
+
     def _add_web_search_tool(
         self,
         responses_api_request: ResponsesAPIOptionalRequestParams,
@@ -1626,7 +1644,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             if response_data.get("usage"):
                 from litellm.responses.utils import ResponseAPILoggingUtils
 
-                usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(response_data.get("usage"))
+                usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
             provider_metadata: Final = _provider_metadata(response_data)
             served_service_tier: Final = response_data.get("service_tier")
             return ModelResponseStream(

litellm/constants.py

@@ -60,6 +60,9 @@ TRACE_READ_RETRY_AFTER_SECONDS: Final = get_env_int("TRACE_READ_RETRY_AFTER_SECO
 OTLP_MAX_CONCURRENT_INGESTS: Final = get_env_int("OTLP_MAX_CONCURRENT_INGESTS", 2)
 AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240)
 AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50)
+LENS_DATASET_MAX_CASES: Final = get_env_int("LENS_DATASET_MAX_CASES", 200)
+LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", 20_000)
+LENS_DATASET_TRACE_PAGE_SIZE: Final = 500
 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))
 DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
 DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16"))
@@ -1074,7 +1077,7 @@ openai_text_completion_compatible_providers: Final[list] = [  # providers that s
     "hyperbolic",
     "wandb",
 ]
-_openai_like_providers: Final[list] = [
+_openai_like_providers: Final[list[str]] = [
     "predibase",
     "databricks",
     "lemonade",
@@ -1807,6 +1810,7 @@ SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL",
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
 SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100")))
+SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, int(os.getenv("SPEND_ROLLUP_LOCK_TIMEOUT_MS", "5000")))
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))

litellm/containers/utils.py

@@ -19,7 +19,7 @@ def decode_managed_container_id_for_request(
     Returns:
         (original_container_id, resolved_provider, updated_litellm_params)
     """
-    decoded: Final = ResponsesAPIRequestUtils._decode_container_id(container_id)
+    decoded: Final = ResponsesAPIRequestUtils.decode_container_id(container_id)
     original_container_id: Final = decoded.get("response_id", container_id)
 
     decoded_provider: Final = decoded.get("custom_llm_provider")
@@ -148,7 +148,7 @@ class ContainerRequestUtils:
 
         # Only encode if we have routing metadata
         if should_encode and response_obj and hasattr(response_obj, "id"):
-            encoded_id: Final = ResponsesAPIRequestUtils._build_container_id(
+            encoded_id: Final = ResponsesAPIRequestUtils.build_container_id(
                 custom_llm_provider=custom_llm_provider,
                 model_id=model_id,
                 container_id=response_obj.id,

litellm/cost_calculator.py

@@ -18,6 +18,7 @@ from litellm.constants import (
     DEFAULT_MAX_LRU_CACHE_SIZE,
     DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND,
 )
+from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
@@ -29,13 +30,13 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     _SERVICE_TIER_TO_COST_KEY_SUFFIX,
     BilledTokenRates,
     CostCalculatorUtils,
-    _generic_cost_per_character,
-    _get_regional_uplift_multiplier,
-    _get_service_tier_cost_key,
     calculate_cost_component,
+    generic_cost_per_character,
     generic_cost_per_token,
     get_batch_cost_rates,
     get_billable_input_tokens,
+    get_regional_uplift_multiplier,
+    get_service_tier_cost_key,
     get_token_type_cost_breakdown,
     parse_prompt_tokens_details,
     select_cost_metric_for_model,
@@ -137,7 +138,7 @@ from litellm.utils import (
     ProviderConfigManager,
     TextCompletionResponse,
     TranscriptionResponse,
-    _cached_get_model_info_helper,
+    cached_get_model_info_helper,
     token_counter,
 )
 
@@ -349,7 +350,7 @@ def _per_second_pricing_cost(
     audio_seconds: float = 0.0,
 ) -> tuple[float, float] | None:
     try:
-        model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
+        model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
     except Exception:  # noqa: BLE001  # the lookup raises plain Exception for an unmapped model
         return None
     if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
@@ -587,7 +588,7 @@ def cost_per_token(
                 raise ValueError(
                     f"prompt_characters must be provided for tts calls. prompt_characters={prompt_characters}, model={model}, custom_llm_provider={custom_llm_provider}, call_type={call_type}"
                 )
-            _prompt_cost, _completion_cost = _generic_cost_per_character(
+            _prompt_cost, _completion_cost = generic_cost_per_character(
                 model=model_without_prefix,
                 custom_llm_provider=custom_llm_provider,
                 prompt_characters=prompt_characters,
@@ -749,7 +750,7 @@ def cost_per_token(
             service_tier=service_tier,
         )
     else:
-        model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
+        model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
         if _has_token_or_tiered_pricing(model_info):
             return generic_cost_per_token(
                 model=model,
@@ -818,7 +819,7 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool:
     )
 
 
-def _select_model_name_for_cost_calc(
+def select_model_name_for_cost_calc(
     model: str | None,
     completion_response: object | None,
     base_model: str | None = None,
@@ -889,6 +890,9 @@ def _select_model_name_for_cost_calc(
     return return_model
 
 
+_select_model_name_for_cost_calc = select_model_name_for_cost_calc
+
+
 def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
     """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
     registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
@@ -1039,9 +1043,9 @@ def get_usage_object(
     elif (
         usage_obj is not None
         and (isinstance(usage_obj, dict) or isinstance(usage_obj, ResponseAPIUsage))
-        and ResponseAPILoggingUtils._is_response_api_usage(usage_obj)
+        and ResponseAPILoggingUtils.is_response_api_usage(usage_obj)
     ):
-        return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage_obj)
+        return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj)
     elif TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj):
         return TranscriptionUsageObjectTransformation.transform_transcription_usage_object(
             cast(
@@ -1070,7 +1074,7 @@ def _is_known_usage_objects(usage_obj):
     )
 
 
-def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: object) -> CallTypesLiteral | None:
+def infer_call_type(call_type: CallTypesLiteral | None, completion_response: object) -> CallTypesLiteral | None:
     if call_type is not None:
         return call_type
 
@@ -1097,6 +1101,9 @@ def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: ob
     return call_type
 
 
+_infer_call_type = infer_call_type
+
+
 def _apply_cost_discount(
     base_cost: float,
     custom_llm_provider: str | None,
@@ -1369,7 +1376,7 @@ def completion_cost(
         - For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
     """
     try:
-        call_type = _infer_call_type(call_type, completion_response) or "completion"
+        call_type = infer_call_type(call_type, completion_response) or "completion"
 
         if call_type == CallTypes.aresponses_websocket.value and isinstance(
             completion_response, LiteLLMRealtimeStreamLoggingObject
@@ -1438,7 +1445,7 @@ def completion_cost(
             )
 
         explicit_pricing: Final = custom_pricing is True or base_model is not None
-        selected_model: Final = _select_model_name_for_cost_calc(
+        selected_model: Final = select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
             custom_llm_provider=custom_llm_provider,
@@ -1492,10 +1499,8 @@ def completion_cost(
                             .calculate_usage(usage_object=_usage, reasoning_content=None)
                             .model_dump()
                         )
-                    elif ResponseAPILoggingUtils._is_response_api_usage(_usage):
-                        _usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
-                            _usage
-                        ).model_dump()
+                    elif ResponseAPILoggingUtils.is_response_api_usage(_usage):
+                        _usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_usage).model_dump()
                     elif TranscriptionUsageObjectTransformation.is_transcription_usage_object(_usage):
                         tr_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(
                             cast(
@@ -1561,7 +1566,7 @@ def completion_cost(
                             "litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - %s",
                             e,
                         )
-                if CostCalculatorUtils._call_type_has_image_response(call_type) and isinstance(
+                if CostCalculatorUtils.call_type_has_image_response(call_type) and isinstance(
                     completion_response, ImageResponse
                 ):
                     ### IMAGE GENERATION COST CALCULATION ###
@@ -1631,7 +1636,7 @@ def completion_cost(
                         video_resolution=video_resolution,
                     )
                 elif call_type in _SPEECH_CALL_TYPES:
-                    prompt_characters = litellm.utils._count_characters(text=prompt)
+                    prompt_characters = litellm.utils.count_characters(text=prompt)
                 elif call_type in _TRANSCRIPTION_CALL_TYPES:
                     # Check _hidden_params first (duration stored there to
                     # avoid polluting the response body), then fall back to
@@ -1780,10 +1785,10 @@ def completion_cost(
                             data={"messages": messages}, call_type="completion"
                         )
 
-                        prompt_characters = litellm.utils._count_characters(text=prompt_string)
+                        prompt_characters = litellm.utils.count_characters(text=prompt_string)
                     if completion_response is not None and isinstance(completion_response, ModelResponse):
                         completion_string = litellm.utils.get_response_string(response_obj=completion_response)
-                        completion_characters = litellm.utils._count_characters(text=completion_string)
+                        completion_characters = litellm.utils.count_characters(text=completion_string)
 
                 # Get the original request model for router detection
                 request_model_for_cost = None
@@ -1946,7 +1951,7 @@ def get_response_cost_from_hidden_params(
     hidden_params: dict | BaseModel,
 ) -> float | None:
     if isinstance(hidden_params, BaseModel):
-        _hidden_params_dict = cast(BaseModel, hidden_params).model_dump()
+        _hidden_params_dict = hidden_params.model_dump()
     else:
         _hidden_params_dict = hidden_params
 
@@ -2024,8 +2029,12 @@ def response_cost_calculator(
             response_cost = 0.0
         else:
             if isinstance(response_object, BaseModel):
-                if hasattr(response_object, "_hidden_params"):
-                    provider_response_cost: Final = get_response_cost_from_hidden_params(response_object._hidden_params)
+                if hasattr(response_object, HIDDEN_PARAMS_ATTR):
+                    hidden_params: Final = cast(  # cast-ok: cost metadata supports dict and Pydantic storage
+                        dict[str, object] | BaseModel,
+                        getattr(response_object, HIDDEN_PARAMS_ATTR),
+                    )
+                    provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
                     if provider_response_cost is not None:
                         return provider_response_cost
 
@@ -2132,7 +2141,7 @@ def pricing_entry_for_cost_calc(
     deployment_key: Final = router_model_id or model
     if deployment_entry is not None and deployment_key is not None:
         return deployment_key, deployment_entry
-    selected_model: Final = _select_model_name_for_cost_calc(
+    selected_model: Final = select_model_name_for_cost_calc(
         model=model,
         completion_response=completion_response,
         base_model=base_model,
@@ -2568,6 +2577,20 @@ def _batch_rate(
     return fallback if rate is None else rate
 
 
+def _batch_or_half(batch_rate: float | None, standard_rate: float | None, default: float = 0.0) -> float:
+    if batch_rate is not None:
+        return batch_rate
+    if standard_rate is not None:
+        return standard_rate / 2
+    return default
+
+
+def _completion_image_tokens(usage: Usage) -> int:
+    details: Final = usage.completion_tokens_details
+    image_tokens: Final = (details.image_tokens or 0) if details is not None else 0
+    return min(image_tokens, usage.completion_tokens)
+
+
 def batch_cost_calculator(
     usage: Usage,
     model: str,
@@ -2607,13 +2630,14 @@ def batch_cost_calculator(
             "output_cost_per_token",
         )
     ):
-        # model_info was provided (e.g. deployment metadata with only id/db_model)
-        # but carries no pricing fields. Fall back to the global pricing table so
-        # that standard model pricing is used instead of silently returning $0.
+        deployment_info: Final = model_info
         try:
             global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
             if global_info:
-                model_info = global_info
+                model_info = {
+                    **global_info,
+                    **{key: value for key, value in deployment_info.items() if value is not None},
+                }
         except Exception:
             pass
 
@@ -2642,19 +2666,22 @@ def batch_cost_calculator(
         )  # batch cost is usually half of the regular token cost
 
         # Add cache read cost if applicable
-        cache_read_cost_key: Final = _get_service_tier_cost_key("cache_read_input_token_cost", None)
+        cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
         total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
 
         cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
         total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
-    if batch_rates.output is not None:
-        total_completion_cost = usage.completion_tokens * batch_rates.output
-    elif output_cost_per_token:
-        total_completion_cost = (
-            usage.completion_tokens * (output_cost_per_token) / 2
-        )  # batch cost is usually half of the regular token cost
+    text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
+    image_rate: Final = _batch_or_half(
+        model_info.get("output_cost_per_image_token_batches"),
+        model_info.get("output_cost_per_image_token"),
+        default=text_rate,
+    )
+    image_tokens: Final = _completion_image_tokens(usage)
+    text_tokens: Final = usage.completion_tokens - image_tokens
+    total_completion_cost = text_tokens * text_rate + image_tokens * image_rate
 
-    uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency)
+    uplift: Final = get_regional_uplift_multiplier(model_info, data_residency)
     if uplift != 1.0:
         total_prompt_cost *= uplift
         total_completion_cost *= uplift
@@ -2809,7 +2836,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
         )
         usage_objects: Final[list[Usage]] = []
         for result in response_done_events:
-            usage_object = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
+            usage_object = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(
                 result["response"].get("usage", {})
             )
             usage_objects.append(usage_object)
@@ -2867,9 +2894,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
         results: Sequence[Mapping[str, object]],
     ) -> tuple[Usage, ...]:
         return tuple(
-            ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(  # pyright: ignore[reportPrivateUsage]  # same shared transform the realtime processor uses
-                response.usage
-            )
+            ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response.usage)
             for _, response in _billable_responses_ws_events(results)
             if response.usage is not None
         )

litellm/decisions/main.py

@@ -206,7 +206,7 @@ def _parse_response(
     response.raise_for_status()
     payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content)
     result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload))
-    result._hidden_params.update(
+    result.hidden_params.update(
         {
             "model": f"{prepared.provider}/{prepared.upstream_model}",
             "custom_llm_provider": prepared.provider,

litellm/exceptions.py

@@ -573,6 +573,54 @@ class ContextWindowExceededError(BadRequestError):
         return _message
 
 
+class PaymentRequiredError(BadRequestError):
+    def __init__(
+        self,
+        message: str,
+        model: str,
+        llm_provider: str,
+        response: httpx.Response | None = None,
+        litellm_debug_info: str | None = None,
+    ) -> None:
+        response_is_valid: Final = (
+            response is not None
+            and isinstance(response, httpx.Response)
+            and hasattr(response, "_request")
+            and getattr(response, "_request", None) is not None
+        )
+        response_for_parent: Final = (
+            response
+            if response_is_valid
+            else httpx.Response(
+                status_code=402,
+                request=httpx.Request(method="GET", url="https://litellm.ai"),
+            )
+        )
+        super().__init__(
+            message=message,
+            model=model,
+            llm_provider=llm_provider,
+            response=response_for_parent,
+            litellm_debug_info=litellm_debug_info,
+        )
+        self.status_code = 402
+        self.message = f"litellm.PaymentRequiredError: {message}"
+
+    def __str__(self) -> str:
+        return (
+            self.message
+            + (f" LiteLLM Retried: {self.num_retries} times" if self.num_retries else "")
+            + (f", LiteLLM Max Retries: {self.max_retries}" if self.max_retries else "")
+        )
+
+    def __repr__(self) -> str:
+        return (
+            self.message
+            + (f" LiteLLM Retried: {self.num_retries} times" if self.num_retries else "")
+            + (f", LiteLLM Max Retries: {self.max_retries}" if self.max_retries else "")
+        )
+
+
 # sub class of bad request error - meant to help us catch guardrails-related errors on proxy.
 class RejectedRequestError(BadRequestError):
     def __init__(
@@ -977,6 +1025,7 @@ LITELLM_EXCEPTION_TYPES: Final = [
     PermissionDeniedError,
     RateLimitError,
     ContextWindowExceededError,
+    PaymentRequiredError,
     RejectedRequestError,
     ContentPolicyViolationError,
     InternalServerError,

litellm/experimental_mcp_client/client.py

@@ -12,7 +12,7 @@ from contextlib import AbstractAsyncContextManager
 from functools import partial
 from importlib.metadata import version
 from types import MappingProxyType
-from typing import Final, TypeAlias, TypeVar, cast
+from typing import TYPE_CHECKING, Final, TypeAlias, TypeVar, cast
 
 import anyio
 import httpx2
@@ -47,9 +47,13 @@ from mcp.types import (
     InitializeRequestParams,
     InitializeResult,
     InputRequiredResult,
+    ListPromptsRequest,
     ListPromptsResult,
+    ListResourcesRequest,
     ListResourcesResult,
     ListResourceTemplatesResult,
+    ListToolsRequest,
+    ListToolsResult,
     PaginatedRequestParams,
     PaginatedResult,
     Prompt,
@@ -89,6 +93,9 @@ from litellm.types.mcp import (
     without_header,
 )
 
+if TYPE_CHECKING:
+    from litellm.proxy._experimental.mcp_server.contracts import CatalogListRequest, CatalogListResult
+
 
 def to_basic_auth(auth_value: str) -> str:
     """Convert auth value to Basic Auth format."""
@@ -830,6 +837,51 @@ class MCPClient:
 
         return factory
 
+    async def list_page(self, request: "CatalogListRequest") -> "CatalogListResult":
+        from mcp.types import INVALID_PARAMS
+
+        params: Final = request.params or PaginatedRequestParams()
+        if isinstance(request, ListToolsRequest):
+            return await self.list_tools_page(params)
+
+        async def fetch(session: ClientSession) -> "CatalogListResult":
+            capabilities: Final = session.server_capabilities
+            empty: Final = (
+                ListPromptsResult(prompts=[])
+                if isinstance(request, ListPromptsRequest)
+                else ListResourcesResult(resources=[])
+                if isinstance(request, ListResourcesRequest)
+                else ListResourceTemplatesResult(resource_templates=[])
+            )
+            supported: Final = capabilities is None or (
+                capabilities.prompts is not None
+                if isinstance(request, ListPromptsRequest)
+                else capabilities.resources is not None
+            )
+            if not supported:
+                if params.cursor is not None:
+                    raise MCPError(
+                        code=INVALID_PARAMS, message="Upstream catalog became unavailable; start a fresh listing"
+                    )
+                return empty
+            try:
+                if isinstance(request, ListPromptsRequest):
+                    return await session.list_prompts(params=params)
+                if isinstance(request, ListResourcesRequest):
+                    return await session.list_resources(params=params)
+                return await session.list_resource_templates(params=params)
+            except MCPError as error:
+                if error.error.code == METHOD_NOT_FOUND and params.cursor is None:
+                    return empty
+                raise
+
+        with anyio.fail_after(max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)):
+            return await self.run_with_session(fetch, quiet_on_error=True)
+
+    async def list_tools_page(self, params: PaginatedRequestParams) -> ListToolsResult:
+        with anyio.fail_after(max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)):
+            return await self.run_with_session(lambda session: session.list_tools(params=params), quiet_on_error=True)
+
     async def list_tools(self, raise_on_error: bool = False) -> list[MCPTool]:
         """List available tools from the server.
 
@@ -846,7 +898,7 @@ class MCPClient:
             # A per-server timeout above the global default extends the whole-walk deadline
             listing_deadline: Final = max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)
             tools: Final = await self.run_with_session(
-                partial(list_tools_with_pagination, listing_deadline=listing_deadline),
+                partial(list_tools_with_pagination, listing_deadline=listing_deadline, require_complete=raise_on_error),
                 quiet_on_error=raise_on_error,
             )
             tool_count: Final = len(tools)

litellm/experimental_mcp_client/tools.py

@@ -2,10 +2,10 @@ import json
 from typing import Final, Literal
 
 import anyio
-from mcp import ClientSession
+from mcp import ClientSession, MCPError
+from mcp.types import INTERNAL_ERROR, PaginatedRequestParams
 from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
 from mcp.types import CallToolResult as MCPCallToolResult
-from mcp.types import PaginatedRequestParams
 from mcp.types import Tool as MCPTool
 from openai.types.chat import ChatCompletionToolParam
 from openai.types.responses.function_tool_param import FunctionToolParam
@@ -99,7 +99,7 @@ def transform_mcp_tool_to_anthropic_tool(mcp_tool: MCPTool) -> AnthropicMessages
 
 
 async def list_tools_with_pagination(
-    session: ClientSession, listing_deadline: float | None = None
+    session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False
 ) -> list[MCPTool]:  # mutable-ok: list return contract
     """Collect tools from every tools/list page by following nextCursor.
 
@@ -137,6 +137,10 @@ async def list_tools_with_pagination(
                     "MCP server repeated a tools/list cursor while listing tools; returning %s tools collected so far",
                     len(tools),
                 )
+                if require_complete:
+                    raise MCPError(
+                        code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: repeated cursor"
+                    )
                 return tools
             seen_cursors.add(next_cursor)
             cursor = next_cursor
@@ -146,6 +150,8 @@ async def list_tools_with_pagination(
             MCP_TOOL_LISTING_MAX_PAGES,
             len(tools),
         )
+        if require_complete:
+            raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: pagination limit")
         return tools
 
     verbose_logger.warning(
@@ -153,6 +159,8 @@ async def list_tools_with_pagination(
         effective_deadline,
         len(tools),
     )
+    if require_complete:
+        raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: listing deadline")
     return tools
 
 

litellm/files/main.py

@@ -30,9 +30,6 @@ FileCreateProvider = Literal[
     "mistral",
     "xai",
 ]
-FileRetrieveProvider = Literal[
-    "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
-]
 FileDeleteProvider = Literal[
     "openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
 ]
@@ -40,7 +37,7 @@ FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthrop
 import litellm
 from litellm import get_secret_str
 from litellm.files.streaming import FileContentStreamingResponse
-from litellm.files.types import FileContentProvider, FileContentStreamingResult
+from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider
 from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj

litellm/files/types.py

@@ -1,9 +1,37 @@
 from collections.abc import AsyncIterator, Iterator, Mapping
-from typing import Literal, NamedTuple
+from typing import Literal, NamedTuple, TypedDict
+
+from typing_extensions import NotRequired, ReadOnly
 
 FileContentProvider = Literal[
     "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "manus", "mistral"
 ]
+FileRetrieveProvider = Literal[
+    "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
+]
+
+
+class FileContentCallOptions(TypedDict, total=False):
+    custom_llm_provider: ReadOnly[FileContentProvider]
+    extra_body: ReadOnly[NotRequired[dict[str, str] | None]]
+    extra_headers: ReadOnly[NotRequired[dict[str, str] | None]]
+    chunk_size: ReadOnly[NotRequired[int]]
+    stream: ReadOnly[NotRequired[bool]]
+
+
+class FileContentRequestKwargs(TypedDict):
+    file_id: ReadOnly[str]
+    custom_llm_provider: ReadOnly[FileContentProvider]
+    extra_body: ReadOnly[NotRequired[dict[str, str] | None]]
+    extra_headers: ReadOnly[NotRequired[dict[str, str] | None]]
+    chunk_size: ReadOnly[NotRequired[int]]
+    stream: ReadOnly[NotRequired[bool]]
+
+
+class FileRetrieveCallOptions(TypedDict, total=False):
+    custom_llm_provider: ReadOnly[FileRetrieveProvider]
+    extra_body: ReadOnly[NotRequired[dict[str, str] | None]]
+    extra_headers: ReadOnly[NotRequired[dict[str, str] | None]]
 
 
 class FileContentStreamingResult(NamedTuple):

litellm/harness/handlers/deepagents_handler.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 
 import asyncio
 import importlib
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Callable, Mapping
 from dataclasses import dataclass
 from types import MappingProxyType, ModuleType
 from typing import TYPE_CHECKING, Any
@@ -59,10 +59,10 @@ _MODEL_NODE = "model"
 class DeepAgentsDeps:
     """The optional-dependency entrypoints this handler uses."""
 
-    create_deep_agent: Any
-    chat_litellm: Any
-    checkpointer_cls: Any
-    command_cls: Any
+    create_deep_agent: Callable[..., CompiledStateGraph]
+    chat_litellm: Callable[..., BaseChatModel]
+    checkpointer_cls: Callable[[], BaseCheckpointSaver]
+    command_cls: Callable[..., Command]
     subagent_defaults: Mapping[str, object]
     convert_to_openai_messages: Any
     backend: ModuleType
@@ -112,7 +112,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
     def __init__(self, config: BaseHarnessConfig) -> None:
         super().__init__(config)
         self._deps: DeepAgentsDeps | None = None
-        self._agent: Any = None
+        self._agent: CompiledStateGraph | None = None
         self._thread_id: str | None = None
         self._skip_tools: frozenset[str] = frozenset()
 
@@ -221,7 +221,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
         if ctx.output is not None:
             ctx.output_json = structured_json(values.get("structured_response"))
 
-    def _require_agent(self) -> tuple[Any, DeepAgentsDeps]:
+    def _require_agent(self) -> tuple[CompiledStateGraph, DeepAgentsDeps]:
         if self._agent is None or self._deps is None:
             raise HarnessError("Deep Agents session is not started")
         return self._agent, self._deps

litellm/images/main.py

@@ -230,7 +230,7 @@ def image_generation(
         else:
             model = "dall-e-2"
             custom_llm_provider = "openai"  # default to dall-e-2 on openai
-        model_response._hidden_params["model"] = model
+        model_response.hidden_params["model"] = model
         openai_params: Final = [
             "user",
             "request_timeout",

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -31,7 +31,7 @@ from litellm.integrations.SlackAlerting.hanging_request_check import (
 )
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 from litellm.litellm_core_utils.exception_mapping_utils import (
-    _add_key_name_and_team_to_alert,
+    add_key_name_and_team_to_alert,
 )
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -293,7 +293,7 @@ class SlackAlerting(CustomBatchLogger):
             # add deployment latencies to alert
             if kwargs is not None and "litellm_params" in kwargs and "metadata" in kwargs["litellm_params"]:
                 _metadata: Final[dict] = kwargs["litellm_params"]["metadata"]
-                request_info = _add_key_name_and_team_to_alert(request_info=request_info, metadata=_metadata)
+                request_info = add_key_name_and_team_to_alert(request_info=request_info, metadata=_metadata)
 
                 _deployment_latency_map: Final = self._get_deployment_latencies_to_alert(metadata=_metadata)
                 if _deployment_latency_map is not None:

litellm/integrations/SlackAlerting/utils.py

@@ -4,7 +4,7 @@ Utils used for slack alerting
 
 import asyncio
 from collections.abc import Callable
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
@@ -57,8 +57,8 @@ def process_slack_alerting_variables(
     return alert_to_webhook_url
 
 
-async def _add_langfuse_trace_id_to_alert(
-    request_data: dict | None = None,
+async def add_langfuse_trace_id_to_alert(
+    request_data: dict[str, object] | None = None,
 ) -> str | None:
     """
     Returns langfuse trace url
@@ -71,7 +71,7 @@ async def _add_langfuse_trace_id_to_alert(
     from litellm.integrations.langfuse.langfuse import LangFuseLogger, resolve_langfuse_host
 
     callbacks: Final[list[CustomLogger | Callable[..., object] | str]] = (
-        litellm.logging_callback_manager._get_all_callbacks()
+        litellm.logging_callback_manager.get_all_callbacks()
     )
     if not any(callback == "langfuse" or isinstance(callback, LangFuseLogger) for callback in callbacks):
         return None
@@ -79,7 +79,9 @@ async def _add_langfuse_trace_id_to_alert(
     if request_data is None or request_data.get("litellm_logging_obj", None) is None:
         return None
 
-    litellm_logging_obj: Final[Logging] = request_data["litellm_logging_obj"]
+    litellm_logging_obj: Final = cast(  # cast-ok: logging object crosses the dynamic callback payload boundary
+        Logging, request_data["litellm_logging_obj"]
+    )
     instance_host: Final = next(
         (callback.langfuse_host for callback in callbacks if isinstance(callback, LangFuseLogger)), None
     )
@@ -87,8 +89,11 @@ async def _add_langfuse_trace_id_to_alert(
         litellm_logging_obj.standard_callback_dynamic_params.get("langfuse_host") or instance_host
     )
     for _ in range(3):
-        if (trace_id := litellm_logging_obj._get_trace_id(service_name="langfuse")) is not None:
+        if (trace_id := litellm_logging_obj.get_trace_id(service_name="langfuse")) is not None:
             return f"{host}/trace/{trace_id}"
         await asyncio.sleep(3)  # wait 3s before retrying for trace id
 
     return None
+
+
+_add_langfuse_trace_id_to_alert = add_langfuse_trace_id_to_alert

litellm/integrations/anthropic_cache_control_hook.py

@@ -283,7 +283,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             openai_dialect
             and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
         ):
-            non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
+            non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="implicit"))
 
         # Points this pass did not place: non-message ones for the provider transform, and
         # the deferred role-targeted ones. Deferring is what reaches the Responses API's

litellm/integrations/arize/arize_phoenix_prompt_manager.py

@@ -118,9 +118,9 @@ class ArizePhoenixTemplateManager:
 
         # Load prompt from Arize Phoenix if prompt_id is provided
         if self.prompt_id:
-            self._load_prompt_from_arize(self.prompt_id)
+            self.load_prompt_from_arize(self.prompt_id)
 
-    def _load_prompt_from_arize(self, prompt_version_id: str) -> None:
+    def load_prompt_from_arize(self, prompt_version_id: str) -> None:
         """Load a specific prompt version from Arize Phoenix."""
         try:
             # Fetch the prompt version from Arize Phoenix
@@ -134,6 +134,8 @@ class ArizePhoenixTemplateManager:
         except Exception as e:
             raise Exception(f"Failed to load prompt version '{prompt_version_id}' from Arize Phoenix: {e}")
 
+    _load_prompt_from_arize = load_prompt_from_arize
+
     def _parse_prompt_data(self, data: dict[str, Any], prompt_version_id: str) -> ArizePhoenixPromptTemplate:
         """Parse Arize Phoenix prompt data and extract messages and metadata."""
         template_data: Final[ArizePhoenixTemplateBody] = data.get("template", {})
@@ -418,7 +420,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
         try:
             # Load the prompt from Arize Phoenix if not already loaded
             if prompt_id not in self.prompt_manager.prompts:
-                self.prompt_manager._load_prompt_from_arize(prompt_id)
+                self.prompt_manager.load_prompt_from_arize(prompt_id)
 
             # Get the rendered messages and metadata
             rendered_messages, prompt_metadata = self.get_prompt_template(prompt_id, prompt_variables)

litellm/integrations/bitbucket/bitbucket_prompt_manager.py

@@ -94,9 +94,9 @@ class BitBucketTemplateManager:
 
         # Load prompts from BitBucket if prompt_id is provided
         if self.prompt_id:
-            self._load_prompt_from_bitbucket(self.prompt_id)
+            self.load_prompt_from_bitbucket(self.prompt_id)
 
-    def _load_prompt_from_bitbucket(self, prompt_id: str) -> None:
+    def load_prompt_from_bitbucket(self, prompt_id: str) -> None:
         """Load a specific .prompt file from BitBucket."""
         try:
             # Fetch the .prompt file from BitBucket
@@ -108,6 +108,8 @@ class BitBucketTemplateManager:
         except Exception as e:
             raise Exception(f"Failed to load prompt '{prompt_id}' from BitBucket: {e}")
 
+    _load_prompt_from_bitbucket = load_prompt_from_bitbucket
+
     def _parse_prompt_file(self, content: str, prompt_id: str) -> BitBucketPromptTemplate:
         """Parse a .prompt file content and extract metadata and template."""
         # Split frontmatter and content
@@ -446,7 +448,7 @@ class BitBucketPromptManager(CustomPromptManagement):
         try:
             # Load the prompt from BitBucket if not already loaded
             if prompt_id not in self.prompt_manager.prompts:
-                self.prompt_manager._load_prompt_from_bitbucket(prompt_id)
+                self.prompt_manager.load_prompt_from_bitbucket(prompt_id)
 
             # Get the rendered prompt and metadata
             rendered_prompt, prompt_metadata = self.get_prompt_template(prompt_id, prompt_variables)

litellm/integrations/custom_logger.py

@@ -991,7 +991,7 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
             import litellm
             from litellm._logging import verbose_logger
 
-            all_callbacks: Final = litellm.logging_callback_manager._get_all_callbacks()
+            all_callbacks: Final = litellm.logging_callback_manager.get_all_callbacks()
 
             for callback_obj in all_callbacks:
                 if hasattr(callback_obj, "increment_callback_logging_failure"):

litellm/integrations/dotprompt/__init__.py

@@ -27,7 +27,7 @@ def set_global_prompt_directory(directory: str) -> None:
     litellm.global_prompt_directory = directory
 
 
-def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict:
+def get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict[str, object]:
     """
     Get the prompt data from the dotprompt content.
 
@@ -37,12 +37,15 @@ def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict:
 
     # Parse the dotprompt content to extract frontmatter and content
     temp_manager: Final = PromptManager()
-    metadata, content = temp_manager._parse_frontmatter(dotprompt_content)
+    metadata, content = temp_manager.parse_frontmatter(dotprompt_content)
 
     # Convert to prompt_data format
     return {"content": content.strip(), "metadata": metadata}
 
 
+_get_prompt_data_from_dotprompt_content = get_prompt_data_from_dotprompt_content
+
+
 def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec") -> "CustomPromptManagement":
     """
     Initialize a prompt from a .prompt file.
@@ -60,7 +63,7 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
     # Handle dotprompt_content from database
     dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None)
     if dotprompt_content and not prompt_data and not prompt_file:
-        prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content)
+        prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content)
 
     from .prompt_manager import strip_version_suffix
 

litellm/integrations/dotprompt/prompt_manager.py

@@ -168,7 +168,7 @@ class PromptManager:
         content: Final = file_path.read_text(encoding="utf-8")
 
         # Split frontmatter and content
-        frontmatter, template_content = self._parse_frontmatter(content)
+        frontmatter, template_content = self.parse_frontmatter(content)
 
         return PromptTemplate(
             content=template_content.strip(),
@@ -176,7 +176,7 @@ class PromptManager:
             template_id=prompt_id,
         )
 
-    def _parse_frontmatter(self, content: str) -> tuple[dict[str, object], str]:
+    def parse_frontmatter(self, content: str) -> tuple[dict[str, object], str]:
         """Parse YAML frontmatter from prompt content."""
         # Match YAML frontmatter between --- delimiters
         frontmatter_pattern: Final = r"^---\s*\n(.*?)\n---\s*\n(.*)$"
@@ -197,6 +197,8 @@ class PromptManager:
 
         return frontmatter, template_content
 
+    _parse_frontmatter = parse_frontmatter
+
     def render(
         self,
         prompt_id: str,
@@ -329,7 +331,7 @@ class PromptManager:
         content: Final = file_path.read_text(encoding="utf-8")
 
         # Parse frontmatter and content
-        frontmatter, template_content = self._parse_frontmatter(content)
+        frontmatter, template_content = self.parse_frontmatter(content)
 
         return {"content": template_content.strip(), "metadata": frontmatter}
 

litellm/integrations/dynamodb.py

@@ -3,7 +3,8 @@
 
 import os
 import traceback
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
+from datetime import datetime
 from typing import Final, Protocol
 
 import litellm
@@ -32,9 +33,18 @@ class DyanmoDBLogger:
             )
         self.table_name = litellm.dynamodb_table_name
 
-    async def _async_log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+    async def async_log_event(
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: datetime,
+        end_time: datetime,
+        print_verbose: Callable[[str], object],
+    ) -> None:
         self.log_event(kwargs, response_obj, start_time, end_time, print_verbose)
 
+    _async_log_event = async_log_event
+
     def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
         try:
             print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")

litellm/integrations/focus/export_engine.py

@@ -2,14 +2,15 @@
 
 from __future__ import annotations
 
-from typing import Any, Final
+from collections.abc import Callable
+from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast
 
 import polars as pl
 
 from litellm._logging import verbose_logger
 
 from .database import FocusLiteLLMDatabase
-from .destinations import FocusDestinationFactory, FocusTimeWindow
+from .destinations import FocusDestination, FocusDestinationFactory, FocusTimeWindow
 from .serializers import FocusCsvSerializer, FocusParquetSerializer, FocusSerializer
 from .transformer import FocusTransformer
 
@@ -28,14 +29,46 @@ class FocusExportEngine:
         self.provider = provider
         self.export_format = export_format
         self.prefix = prefix
-        self._destination = FocusDestinationFactory.create(
+        self.destination = FocusDestinationFactory.create(
             provider=self.provider,
             prefix=self.prefix,
             config=destination_config,
         )
-        self._serializer = self._init_serializer()
-        self._transformer = FocusTransformer()
-        self._database = FocusLiteLLMDatabase()
+        self.serializer = self._init_serializer()
+        self.transformer = FocusTransformer()
+        self.database = FocusLiteLLMDatabase()
+
+    @property
+    def _destination(self) -> FocusDestination:
+        return self.destination
+
+    @_destination.setter
+    def _destination(self, value: FocusDestination) -> None:
+        self.destination = value
+
+    @property
+    def _serializer(self) -> FocusSerializer:
+        return self.serializer
+
+    @_serializer.setter
+    def _serializer(self, value: FocusSerializer) -> None:
+        self.serializer = value
+
+    @property
+    def _transformer(self) -> FocusTransformer:
+        return self.transformer
+
+    @_transformer.setter
+    def _transformer(self, value: FocusTransformer) -> None:
+        self.transformer = value
+
+    @property
+    def _database(self) -> FocusLiteLLMDatabase:
+        return self.database
+
+    @_database.setter
+    def _database(self, value: FocusLiteLLMDatabase) -> None:
+        self.database = value
 
     def _init_serializer(self) -> FocusSerializer:
         if self.export_format == "csv":
@@ -45,18 +78,18 @@ class FocusExportEngine:
         raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.")
 
     async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]:
-        data: Final = await self._database.get_usage_data(limit=limit)
-        normalized: Final = self._transformer.transform(data)
+        data: Final = await self.database.get_usage_data(limit=limit)
+        normalized: Final = self.transformer.transform(data)
 
         usage_sample: Final = data.head(min(50, len(data))).to_dicts()
         normalized_sample: Final = normalized.head(min(50, len(normalized))).to_dicts()
 
         summary: Final = {
             "total_records": len(normalized),
-            "total_spend": self._sum_column(data, "spend"),
-            "total_tokens": self._sum_column(data, "total_tokens"),
-            "unique_teams": self._count_unique(data, "team_id"),
-            "unique_models": self._count_unique(data, "model"),
+            "total_spend": self.sum_column(data, "spend"),
+            "total_tokens": self.sum_column(data, "total_tokens"),
+            "unique_teams": self.count_unique(data, "team_id"),
+            "unique_models": self.count_unique(data, "model"),
         }
 
         return {
@@ -71,12 +104,12 @@ class FocusExportEngine:
         limit: int | None,
     ) -> None:
         """Export all available data without time-window filtering."""
-        data: Final = await self._database.get_usage_data(limit=limit)
+        data: Final = await self.database.get_usage_data(limit=limit)
         if data.is_empty():
             verbose_logger.debug("Focus export: no usage data available")
             return
 
-        normalized: Final = self._transformer.transform(data)
+        normalized: Final = self.transformer.transform(data)
         if normalized.is_empty():
             verbose_logger.debug("Focus export: normalized data empty")
             return
@@ -98,7 +131,7 @@ class FocusExportEngine:
         window: FocusTimeWindow,
         limit: int | None,
     ) -> None:
-        data: Final = await self._database.get_usage_data(
+        data: Final = await self.database.get_usage_data(
             limit=limit,
             start_time_utc=window.start_time,
             end_time_utc=window.end_time,
@@ -107,7 +140,7 @@ class FocusExportEngine:
             verbose_logger.debug("Focus export: no usage data for window %s", window)
             return
 
-        normalized: Final = self._transformer.transform(data)
+        normalized: Final = self.transformer.transform(data)
         if normalized.is_empty():
             verbose_logger.debug("Focus export: normalized data empty for window %s", window)
             return
@@ -115,39 +148,45 @@ class FocusExportEngine:
         await self._serialize_and_upload(normalized, window)
 
     async def _serialize_and_upload(self, frame: pl.DataFrame, window: FocusTimeWindow) -> None:
-        payload: Final = self._serializer.serialize(frame)
+        payload: Final = self.serializer.serialize(frame)
         if not payload:
             verbose_logger.debug("Focus export: serializer returned empty payload")
             return
-        await self._destination.deliver(
+        await self.destination.deliver(
             content=payload,
             time_window=window,
-            filename=self._build_filename(window),
+            filename=self.build_filename(window),
         )
 
-    def _build_filename(self, window: FocusTimeWindow) -> str:
-        if not self._serializer.extension:
+    def build_filename(self, window: FocusTimeWindow) -> str:
+        if not self.serializer.extension:
             raise ValueError("Serializer must declare a file extension")
         # Include time window in filename so Vantage (which deduplicates
         # by filename) doesn't overwrite previous uploads.
         start_str: Final = window.start_time.strftime("%Y%m%dT%H%M%SZ")
         end_str: Final = window.end_time.strftime("%Y%m%dT%H%M%SZ")
-        return f"usage_{start_str}_{end_str}.{self._serializer.extension}"
+        return f"usage_{start_str}_{end_str}.{self.serializer.extension}"
+
+    _build_filename = build_filename
 
     @staticmethod
-    def _sum_column(frame: pl.DataFrame, column: str) -> float:
+    def sum_column(frame: pl.DataFrame, column: str) -> float:
         if frame.is_empty() or column not in frame.columns:
             return 0.0
-        value: Final = frame.select(pl.col(column).sum().alias("sum")).row(0)[0]
+        value: Final[object] = frame.select(pl.col(column).sum().alias("sum")).row(0)[0]
         if value is None:
             return 0.0
-        return float(value)
+        return float(cast(SupportsFloat | SupportsIndex | str | bytes | bytearray, value))
+
+    _sum_column: Final[Callable[[pl.DataFrame, str], float]] = sum_column
 
     @staticmethod
-    def _count_unique(frame: pl.DataFrame, column: str) -> int:
+    def count_unique(frame: pl.DataFrame, column: str) -> int:
         if frame.is_empty() or column not in frame.columns:
             return 0
-        value: Final = frame.select(pl.col(column).n_unique().alias("unique")).row(0)[0]
+        value: Final[object] = frame.select(pl.col(column).n_unique().alias("unique")).row(0)[0]
         if value is None:
             return 0
-        return int(value)
+        return int(cast(SupportsInt | SupportsIndex | str | bytes | bytearray, value))
+
+    _count_unique: Final[Callable[[pl.DataFrame, str], int]] = count_unique

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -120,17 +120,19 @@ class GitLabTemplateManager:
         )
 
         if self.prompt_id:
-            self._load_prompt_from_gitlab(self.prompt_id)
+            self.load_prompt_from_gitlab(self.prompt_id)
 
     # ---------- path helpers ----------
 
-    def _id_to_repo_path(self, prompt_id: str) -> str:
+    def id_to_repo_path(self, prompt_id: str) -> str:
         """Map a prompt_id to a repo path (respects prompts_path and adds .prompt)."""
         prompt_id = decode_prompt_id(prompt_id)
         if self.prompts_path:
             return f"{self.prompts_path}/{prompt_id}.prompt"
         return f"{prompt_id}.prompt"
 
+    _id_to_repo_path = id_to_repo_path
+
     def _repo_path_to_id(self, repo_path: str) -> str:
         """
         Map a repo path like 'prompts/chat/greeting.prompt' to an ID relative
@@ -144,11 +146,11 @@ class GitLabTemplateManager:
 
     # ---------- loading ----------
 
-    def _load_prompt_from_gitlab(self, prompt_id: str, *, ref: str | None = None) -> None:
+    def load_prompt_from_gitlab(self, prompt_id: str, *, ref: str | None = None) -> None:
         """Load a specific .prompt file from GitLab (scoped under prompts_path if set)."""
         try:
             # prompt_id = decode_prompt_id(prompt_id)
-            file_path: Final = self._id_to_repo_path(prompt_id)
+            file_path: Final = self.id_to_repo_path(prompt_id)
             prompt_content: Final = self.gitlab_client.get_file_content(file_path, ref=ref)
             if prompt_content:
                 template: Final = self._parse_prompt_file(prompt_content, prompt_id)
@@ -156,6 +158,8 @@ class GitLabTemplateManager:
         except Exception as e:
             raise Exception(f"Failed to load prompt '{encode_prompt_id(prompt_id)}' from GitLab: {e}")
 
+    _load_prompt_from_gitlab = load_prompt_from_gitlab
+
     def load_all_prompts(self, *, recursive: bool = True) -> list[str]:
         """
         Eagerly load all .prompt files from prompts_path. Returns loaded IDs.
@@ -164,7 +168,7 @@ class GitLabTemplateManager:
         loaded: Final[list[str]] = []
         for pid in files:
             if pid not in self.prompts:
-                self._load_prompt_from_gitlab(pid)
+                self.load_prompt_from_gitlab(pid)
             loaded.append(pid)
         return loaded
 
@@ -333,7 +337,7 @@ class GitLabPromptManager(CustomPromptManagement):
         ref: str | None = None,
     ) -> tuple[str, dict[str, Any]]:
         if prompt_id not in self.prompt_manager.prompts:
-            self.prompt_manager._load_prompt_from_gitlab(prompt_id, ref=ref)
+            self.prompt_manager.load_prompt_from_gitlab(prompt_id, ref=ref)
 
         template: Final = self.prompt_manager.get_template(prompt_id)
         if not template:
@@ -506,7 +510,7 @@ class GitLabPromptManager(CustomPromptManagement):
                     if hasattr(dynamic_callback_params, "extra")
                     else None
                 )
-                self.prompt_manager._load_prompt_from_gitlab(decoded_id, ref=git_ref)
+                self.prompt_manager.load_prompt_from_gitlab(decoded_id, ref=git_ref)
 
             rendered_prompt, prompt_metadata = self.get_prompt_template(prompt_id, prompt_variables)
 
@@ -689,17 +693,17 @@ class GitLabPromptCache:
         for pid in ids:
             # Ensure template is loaded into TemplateManager
             if pid not in self.template_manager.prompts:
-                self.template_manager._load_prompt_from_gitlab(pid)
+                self.template_manager.load_prompt_from_gitlab(pid)
 
             tmpl = self.template_manager.get_template(pid)
             if tmpl is None:
                 # If something raced/failed, try once more
-                self.template_manager._load_prompt_from_gitlab(pid)
+                self.template_manager.load_prompt_from_gitlab(pid)
                 tmpl = self.template_manager.get_template(pid)
             if tmpl is None:
                 continue
 
-            file_path = self.template_manager._id_to_repo_path(pid)  # "prompts/chat/..../file.prompt"
+            file_path = self.template_manager.id_to_repo_path(pid)  # "prompts/chat/..../file.prompt"
             entry = self._template_to_json(pid, tmpl)
 
             self._by_file[file_path] = entry
@@ -758,7 +762,7 @@ class GitLabPromptCache:
 
         return {
             "id": prompt_id,  # e.g. "greet/hi"
-            "path": self.template_manager._id_to_repo_path(prompt_id),  # e.g. "prompts/chat/greet/hi.prompt"
+            "path": self.template_manager.id_to_repo_path(prompt_id),  # e.g. "prompts/chat/greet/hi.prompt"
             "content": tmpl.content,  # rendered content (without frontmatter)
             "metadata": md,  # parsed frontmatter
             "model": model,

litellm/integrations/langfuse/langfuse.py

@@ -1023,7 +1023,7 @@ class LangFuseLogger:
                 _cache_key = _hidden_params.get("cache_key", None)
                 if _cache_key is None and litellm.cache is not None:
                     # fallback to using "preset_cache_key"
-                    _preset_cache_key: Final = litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)  # pyright: ignore[reportPrivateUsage]  # kwargs-ok: no public preset-cache-key accessor
+                    _preset_cache_key: Final = litellm.cache.get_preset_cache_key_from_kwargs(**kwargs)
                     _cache_key = _preset_cache_key
                 tags.append(f"cache_key:{_cache_key}")
         return tags

litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py

@@ -116,7 +116,7 @@ class MavvrikFocusLogger(FocusLogger):
         """Export with Mavvrik row cap applied when no explicit limit is passed."""
         effective_limit: Final = limit if limit is not None else self._max_rows
         engine: Final = self._ensure_engine()
-        data: Final = await engine._database.get_usage_data(
+        data: Final = await engine.database.get_usage_data(
             limit=effective_limit,
             start_time_utc=window.start_time,
             end_time_utc=window.end_time,
@@ -134,13 +134,13 @@ class MavvrikFocusLogger(FocusLogger):
         if data.is_empty():
             verbose_proxy_logger.debug("Mavvrik FOCUS export: no usage data for window %s", window)
         else:
-            normalized: Final = engine._transformer.transform(data)
+            normalized: Final = engine.transformer.transform(data)
             if not normalized.is_empty():
-                payload = engine._serializer.serialize(normalized)
-        await engine._destination.deliver(
+                payload = engine.serializer.serialize(normalized)
+        await engine.destination.deliver(
             content=payload or b"",
             time_window=window,
-            filename=engine._build_filename(window),
+            filename=engine.build_filename(window),
         )
 
     # Maximum number of days to catch up in a single run. Prevents runaway
@@ -165,7 +165,7 @@ class MavvrikFocusLogger(FocusLogger):
             FocusMavvrikDestination,
         )
 
-        destination: Final = engine._destination
+        destination: Final = engine.destination
         if not isinstance(destination, FocusMavvrikDestination):
             await super()._run_scheduled_export()
             return

litellm/integrations/opentelemetry.py

@@ -32,6 +32,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.metadata import flatten_metadata
 from litellm.integrations.otel.model.semconv import LiteLLM, Metric
 from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
+from litellm.integrations.otel.routing import routing_decision_attributes
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.secret_redaction import redact_string
@@ -180,7 +181,7 @@ class OTELMetricAttributeFilter:
     exclude_list: list[str] | None = None
 
 
-def _build_metric_attribute_filter(value: object) -> OTELMetricAttributeFilter:
+def build_metric_attribute_filter(value: object) -> OTELMetricAttributeFilter:
     if isinstance(value, OTELMetricAttributeFilter):
         return value
     if not isinstance(value, dict):
@@ -194,7 +195,10 @@ def _build_metric_attribute_filter(value: object) -> OTELMetricAttributeFilter:
     )
 
 
-def _resolve_metric_attribute_filter(
+_build_metric_attribute_filter = build_metric_attribute_filter
+
+
+def resolve_metric_attribute_filter(
     attributes: OTELMetricAttributeFilter | None,
 ) -> tuple[frozenset[str] | None, frozenset[str] | None]:
     if attributes is None:
@@ -219,6 +223,9 @@ def _resolve_metric_attribute_filter(
     )
 
 
+_resolve_metric_attribute_filter = resolve_metric_attribute_filter
+
+
 def _provider_label(custom_llm_provider: object) -> str | None:
     """The provider label for one call's metrics and events, or None when the
     call carries no provider.
@@ -407,7 +414,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         if metadata_keys_override is not None:
             config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override)
         if metric_attributes_override is not None:
-            config.attributes = _build_metric_attribute_filter(metric_attributes_override)
+            config.attributes = build_metric_attribute_filter(metric_attributes_override)
 
         self.config = config
         self.callback_name = callback_name
@@ -1642,11 +1649,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             otel_settings: Final = (litellm.callback_settings or {}).get("otel") or {}
             raw: Final[object] = otel_settings.get("attributes") if isinstance(otel_settings, dict) else None
             if raw is not None:
-                attributes = _build_metric_attribute_filter(raw)
+                attributes = build_metric_attribute_filter(raw)
         (
             self._metric_attr_include,
             self._metric_attr_exclude,
-        ) = _resolve_metric_attribute_filter(attributes)
+        ) = resolve_metric_attribute_filter(attributes)
         self._metric_attr_filter_resolved = True
 
     def _filter_metric_attributes(self, attrs: Mapping[str, str | None]) -> dict[str, str]:
@@ -2407,6 +2414,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             metadata: Final = standard_logging_payload["metadata"]
             for key, value in metadata.items():
                 self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value)
+            decision: Final = metadata.get("routing_decision")
+            span.set_attributes(routing_decision_attributes(decision))
 
             # get hidden params
             hidden_params: Final = getattr(standard_logging_payload, "hidden_params", None) or (

litellm/integrations/otel/logger.py

@@ -54,6 +54,7 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic
 from litellm.integrations.otel.model.utils import to_ns
 from litellm.integrations.otel.plumbing.context import (
     active_phase,
+    active_phase_span,
     is_recordable_span,
     mcp_message_transport_span,
     post_response_root,
@@ -778,6 +779,11 @@ class OpenTelemetryV2(CustomLogger):
         if is_recordable_span(span):
             span.add_event(name, attributes)
 
+    def set_phase_attributes(self, attributes: Mapping[str, str | int | float | bool]) -> None:
+        span: Final = active_phase_span()
+        if span is not None and is_recordable_span(span):
+            span.set_attributes(attributes)
+
     async def async_pre_call_hook(
         self,
         user_api_key_dict: "UserAPIKeyAuth",
@@ -1042,6 +1048,12 @@ def phase_event(name: str, attributes: Mapping[str, str | int] | None = None) ->
         logger.add_phase_event(name, attributes)
 
 
+def phase_attributes(attributes: Mapping[str, str | int | float | bool]) -> None:
+    logger: Final = _registered_v2_logger()
+    if logger is not None:
+        logger.set_phase_attributes(attributes)
+
+
 def build_otel_v2_logger(
     config: OpenTelemetryV2Config,
     callback_name: str | None = None,

litellm/integrations/otel/mappers/genai.py

@@ -174,7 +174,7 @@ class GenAIMapper:
                 return {}
 
     def _llm_call(self, data: LLMCallSpanData) -> AttributeMap:
-        attrs: Final = collect(self._LLM_CALL_ATTRS, data)
+        attrs: Final = {**collect(self._LLM_CALL_ATTRS, data), **data.routing_attributes}
         if data.tools:
             attrs[LiteLLM.TOOLS_DECLARED] = len(data.tools)
             attrs.update(

litellm/integrations/otel/model/payloads.py

@@ -30,6 +30,7 @@ from litellm.integrations.otel.model.utils import (
     as_str_mapping,
     as_str_tuple,
 )
+from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes
 
 # ``RequestIdentity`` and the request-metadata translation now live in
 # :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep
@@ -436,6 +437,7 @@ class LLMCallSpanData:
     trace: TraceControls = field(default_factory=TraceControls)
     session_id: str | None = None
     embedding_output: EmbeddingOutput | None = None
+    routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
 
     @classmethod
     def from_standard_logging_payload(
@@ -453,6 +455,8 @@ class LLMCallSpanData:
         # model split, the response model, api base, and identity all come from
         # here rather than being re-derived from the raw payload dicts.
         context: Final = RequestContext.from_standard_logging_payload(payload)
+        metadata: Final = payload.get("metadata")
+        decision: Final = metadata["routing_decision"] if metadata and "routing_decision" in metadata else None
         # Normalize ``response`` to a dict once so the content/id reads below are a
         # plain ``.get`` — no repeated ``isinstance`` guards.
         raw_response: Final = payload.get("response")
@@ -496,6 +500,7 @@ class LLMCallSpanData:
             trace=trace or TraceControls(),
             session_id=session_id or None,
             embedding_output=embedding_output if capture_content else None,
+            routing_attributes=routing_decision_attributes(decision),
         )
 
 

litellm/integrations/otel/model/spans.py

@@ -349,6 +349,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_WorkflowEvent",
         "LiteLLM_WorkflowMessage",
         "LiteLLM_Lens",
+        "LiteLLM_LensDataset",
         "LiteLLM_LensRun",
         "LiteLLM_LensReview",
         "LiteLLM_LensWorker",

litellm/integrations/otel/plumbing/metrics.py

@@ -21,8 +21,8 @@ from litellm._logging import verbose_logger
 from litellm.integrations.opentelemetry import (
     METRIC_METADATA_KEYS,
     TOKEN_TYPE_ATTRIBUTE,
-    _build_metric_attribute_filter,
-    _resolve_metric_attribute_filter,
+    build_metric_attribute_filter,
+    resolve_metric_attribute_filter,
 )
 from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds
 from litellm.integrations.otel.model.semconv import (
@@ -324,13 +324,13 @@ class GenAIMetricRecorder:
             otel_settings: Final = (litellm.callback_settings or {}).get("otel") or {}
             raw: Final[object] = otel_settings.get("attributes") if isinstance(otel_settings, dict) else None
             if raw is not None:
-                attributes = _build_metric_attribute_filter(raw)
+                attributes = build_metric_attribute_filter(raw)
         # A bad filter (include_list + exclude_list both set, an unfilterable name)
         # raises here; the caller (logger._record_metrics) surfaces it once at ERROR
         # so the operator-fixable config error is visible. Not cached on the raise
         # path -- _filter_resolved stays False -- so a corrected config takes effect
         # without reconstructing the recorder.
-        self._include, self._exclude = _resolve_metric_attribute_filter(attributes)
+        self._include, self._exclude = resolve_metric_attribute_filter(attributes)
         self._filter_resolved = True
         self._warn_about_metric_ineligible_names()
 

litellm/integrations/otel/presets/weave.py

@@ -12,7 +12,7 @@ from litellm.integrations.otel.presets.utils import (
     ensure_mappers,
 )
 from litellm.integrations.weave.weave_otel import (
-    _get_weave_authorization_header,
+    get_weave_authorization_header,
     get_weave_otel_config,
 )
 from litellm.types.utils import StandardCallbackDynamicParams
@@ -58,7 +58,7 @@ def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, st
     headers: Final[dict[str, str]] = {}
     api_key: Final = params.get("wandb_api_key")
     if api_key:
-        headers["Authorization"] = _get_weave_authorization_header(api_key=api_key)
+        headers["Authorization"] = get_weave_authorization_header(api_key=api_key)
     project_id: Final = params.get("weave_project_id")
     if project_id:
         headers["project_id"] = project_id

litellm/integrations/otel/routing.py

@@ -0,0 +1,57 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, TypeAlias
+
+RoutingAttributeValue: TypeAlias = str | int | float | bool
+
+_ROUTING_SCALAR_FIELDS: Final = frozenset(
+    {
+        "router_model_name",
+        "router_type",
+        "router_config_id",
+        "router_config_updated_at",
+        "router_config_fingerprint",
+        "routed_model",
+        "cause",
+        "tier",
+        "tier_label",
+        "request_type",
+        "score",
+        "classifier_model",
+        "classifier_cost",
+        "classifier_failure_reason",
+        "classifier_error_type",
+        "classifier_confidence",
+        "classifier_primary_rule",
+        "classifier_capability_boundary",
+        "classifier_p_solve",
+        "classifier_calibrated_p_solve",
+        "classifier_calibration_version",
+        "classifier_efficient_p_solve",
+        "classifier_capable_p_solve",
+        "classifier_calibrated_efficient_p_solve",
+        "classifier_calibrated_capable_p_solve",
+        "classifier_max_quality_gap",
+        "classifier_prompt_version",
+        "classifier_threshold",
+        "escalated",
+        "context_escalated",
+        "context_escalation_original_tier",
+        "reasoning_override_min_score",
+        "conversation_continuing",
+        "savings_baseline_model",
+        "savings_baseline_deployment_id",
+    }
+)
+
+
+def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mapping[str, RoutingAttributeValue]:
+    if not isinstance(decision, Mapping):
+        return MappingProxyType({})
+    return MappingProxyType(
+        {
+            f"litellm.routing.{key}": value
+            for key in _ROUTING_SCALAR_FIELDS
+            if isinstance(value := decision.get(key), (str, int, float, bool))
+        }
+    )

litellm/integrations/otel/runtime.py

@@ -7,7 +7,9 @@ V2 is not the active logger — so a call site can wrap a request phase or seed
 identity unconditionally.
 """
 
-from collections.abc import Callable, Iterator, Mapping
+from __future__ import annotations
+
+from collections.abc import Callable, Generator, Mapping
 from contextlib import AbstractContextManager, contextmanager
 from functools import cache
 from typing import TYPE_CHECKING, Final, TypeAlias
@@ -16,11 +18,18 @@ if TYPE_CHECKING:
     from opentelemetry.trace import Span
 
 PhaseEventAttributes: TypeAlias = Mapping[str, str | int]
+PhaseAttributes: TypeAlias = Mapping[str, str | int | float | bool]
 
 
 @cache
 def _otel_runtime() -> (
-    "tuple[Callable[[str], AbstractContextManager[Span | None]], Callable[..., None], Callable[[str, PhaseEventAttributes | None], None]] | None"
+    tuple[
+        Callable[[str], AbstractContextManager[Span | None]],
+        Callable[..., None],
+        Callable[[str, PhaseEventAttributes | None], None],
+        Callable[[PhaseAttributes], None],
+    ]
+    | None
 ):
     """Resolve the SDK-backed hooks once and cache the outcome, absence included.
 
@@ -32,11 +41,11 @@ def _otel_runtime() -> (
         from litellm.integrations.otel import logger
     except Exception:
         return None
-    return (logger.phase_span, logger.seed_request_identity, logger.phase_event)
+    return (logger.phase_span, logger.seed_request_identity, logger.phase_event, logger.phase_attributes)
 
 
 @contextmanager
-def phase_span(name: str) -> "Iterator[Span | None]":
+def phase_span(name: str) -> Generator[Span | None]:
     """Run a request phase inside a live active span so its DB/service calls nest.
 
     Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
@@ -64,3 +73,10 @@ def seed_request_identity(user_api_key_dict: object, model: object = None) -> No
     if runtime is None:
         return
     runtime[1](user_api_key_dict, model=model)
+
+
+def phase_attributes(attributes: PhaseAttributes) -> None:
+    runtime: Final = _otel_runtime()
+    if runtime is None:
+        return
+    runtime[3](attributes)

litellm/integrations/prometheus.py

@@ -32,7 +32,7 @@ from litellm.exceptions import (
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.prometheus_helpers import (
     PrometheusLabelFactoryContext,
-    _get_cached_end_user_id_for_cost_tracking,
+    get_cached_end_user_id_for_cost_tracking,
 )
 from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
     BoundedPrometheusSeriesTracker,
@@ -65,8 +65,8 @@ from litellm.repositories.user_repository import UserRepository
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
-    _sanitize_prometheus_label_name,
-    _sanitize_prometheus_label_value,
+    sanitize_prometheus_label_name,
+    sanitize_prometheus_label_value,
     validate_prometheus_deployment_and_latency_caller_identity,
 )
 from litellm.types.proxy.carried_budget_state import (
@@ -1069,10 +1069,10 @@ class PrometheusLogger(CustomLogger):
 
         builtin_labels: Final = frozenset(label.value for label in UserAPIKeyLabelNames)
         custom_metadata_labels: Final = frozenset(
-            _sanitize_prometheus_label_name(label) for label in litellm.custom_prometheus_metadata_labels
+            sanitize_prometheus_label_name(label) for label in litellm.custom_prometheus_metadata_labels
         )
         custom_tag_labels: Final = frozenset(
-            _sanitize_prometheus_label_name(f"tag_{tag}") for tag in litellm.custom_prometheus_tags
+            sanitize_prometheus_label_name(f"tag_{tag}") for tag in litellm.custom_prometheus_tags
         )
         return builtin_labels | _NON_ENUM_METRIC_LABELS | custom_metadata_labels | custom_tag_labels
 
@@ -1508,7 +1508,7 @@ class PrometheusLogger(CustomLogger):
         model: Final = kwargs.get("model", "")
         litellm_params: Final = kwargs.get("litellm_params", {}) or {}
         _metadata: Final = litellm_params.get("metadata") or {}
-        get_end_user_id_for_cost_tracking: Final = _get_cached_end_user_id_for_cost_tracking()
+        get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking()
 
         end_user_id: Final = get_end_user_id_for_cost_tracking(litellm_params, service_type="prometheus")
         user_id: Final = standard_logging_payload["metadata"]["user_api_key_user_id"]
@@ -2523,7 +2523,7 @@ class PrometheusLogger(CustomLogger):
         model: Final = kwargs.get("model", "")
 
         litellm_params: Final = kwargs.get("litellm_params", {}) or {}
-        get_end_user_id_for_cost_tracking: Final = _get_cached_end_user_id_for_cost_tracking()
+        get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking()
 
         end_user_id: Final = get_end_user_id_for_cost_tracking(litellm_params, service_type="prometheus")
         user_id: Final = standard_logging_payload["metadata"]["user_api_key_user_id"]
@@ -2775,7 +2775,7 @@ class PrometheusLogger(CustomLogger):
         status_code: Final = self._extract_status_code(exception=original_exception)
 
         try:
-            _tags: Final = StandardLoggingPayloadSetup._get_request_tags(
+            _tags: Final = StandardLoggingPayloadSetup.get_request_tags(
                 litellm_params=request_data,
                 proxy_server_request=request_data.get("proxy_server_request", {}),
             )
@@ -3725,11 +3725,11 @@ class PrometheusLogger(CustomLogger):
         increment metric when litellm.Router / load balancing logic places a deployment in cool down
         """
         self.litellm_deployment_cooled_down.labels(
-            _sanitize_prometheus_label_value(litellm_model_name),
-            _sanitize_prometheus_label_value(model_id),
-            _sanitize_prometheus_label_value(api_base),
-            _sanitize_prometheus_label_value(api_provider),
-            _sanitize_prometheus_label_value(exception_status),
+            sanitize_prometheus_label_value(litellm_model_name),
+            sanitize_prometheus_label_value(model_id),
+            sanitize_prometheus_label_value(api_base),
+            sanitize_prometheus_label_value(api_provider),
+            sanitize_prometheus_label_value(exception_status),
         ).inc()
 
     def increment_callback_logging_failure(
@@ -4750,17 +4750,17 @@ def _prometheus_labels_from_context(
     ctx: PrometheusLabelFactoryContext,
 ) -> dict[str, str | None]:
     filtered_labels: Final[dict[str, str | None]] = {
-        label: ctx._sanitized_enum[label] for label in supported_enum_labels if label in ctx._sanitized_enum
+        label: ctx.sanitized_enum[label] for label in supported_enum_labels if label in ctx.sanitized_enum
     }
 
     if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
         filtered_labels[UserAPIKeyLabelNames.END_USER.value] = ctx.get_resolved_end_user()
 
-    for sk, val in ctx._custom_by_sanitized_key.items():
+    for sk, val in ctx.custom_by_sanitized_key.items():
         if sk in supported_enum_labels:
             filtered_labels[sk] = val
 
-    for k, v in ctx._tag_labels.items():
+    for k, v in ctx.tag_labels.items():
         if k in supported_enum_labels:
             filtered_labels[k] = v
 
@@ -4797,13 +4797,13 @@ def prometheus_label_factory(
     # Filter supported labels and sanitize values to prevent breaking
     # the Prometheus text format (e.g. U+2028 Line Separator in label values)
     filtered_labels: Final = {
-        label: _sanitize_prometheus_label_value(value)
+        label: sanitize_prometheus_label_value(value)
         for label, value in enum_dict.items()
         if label in supported_enum_labels
     }
 
     if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
-        get_end_user_id_for_cost_tracking: Final = _get_cached_end_user_id_for_cost_tracking()
+        get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking()
 
         filtered_labels["end_user"] = get_end_user_id_for_cost_tracking(
             litellm_params={"user_api_key_end_user_id": enum_values.end_user},
@@ -4813,16 +4813,16 @@ def prometheus_label_factory(
     if enum_values.custom_metadata_labels is not None:
         for key, value in enum_values.custom_metadata_labels.items():
             # check sanitized key
-            sanitized_key = _sanitize_prometheus_label_name(key)
+            sanitized_key = sanitize_prometheus_label_name(key)
             if sanitized_key in supported_enum_labels:
-                filtered_labels[sanitized_key] = _sanitize_prometheus_label_value(value)
+                filtered_labels[sanitized_key] = sanitize_prometheus_label_value(value)
 
     # Add custom tags if configured
     if enum_values.tags is not None:
         custom_tag_labels: Final = get_custom_labels_from_tags(enum_values.tags)
         for key, value in custom_tag_labels.items():
             if key in supported_enum_labels:
-                filtered_labels[key] = _sanitize_prometheus_label_value(value)
+                filtered_labels[key] = sanitize_prometheus_label_value(value)
 
     for label in supported_enum_labels:
         if label not in filtered_labels:
@@ -4919,7 +4919,7 @@ def _tag_matches_wildcard_configured_pattern(tags: Sequence[str], configured_tag
     from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
 
     pattern_router: Final = PatternMatchRouter()
-    regex_pattern: Final = pattern_router._pattern_to_regex(configured_tag)
+    regex_pattern: Final = pattern_router.pattern_to_regex(configured_tag)
     return any(re.match(pattern=regex_pattern, string=tag) for tag in tags)
 
 
@@ -4945,7 +4945,7 @@ def get_custom_labels_from_tags(tags: Sequence[str]) -> dict[str, str]:
     }
     """
 
-    from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
+    from litellm.types.integrations.prometheus import sanitize_prometheus_label_name
 
     configured_tags: Final = litellm.custom_prometheus_tags
     if configured_tags is None or len(configured_tags) == 0:
@@ -4954,7 +4954,7 @@ def get_custom_labels_from_tags(tags: Sequence[str]) -> dict[str, str]:
     result: Final[dict[str, str]] = {}
 
     for configured_tag in configured_tags:
-        label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}")
+        label_name = sanitize_prometheus_label_name(f"tag_{configured_tag}")
 
         # Check for exact match first (backwards compatibility)
         if configured_tag in tags:

litellm/integrations/prometheus_helpers/__init__.py

@@ -6,18 +6,26 @@ Helpers for the Prometheus integration (extracted to keep ``prometheus.py`` smal
 
 from __future__ import annotations
 
-from typing import Final, cast
+from typing import Final, Literal, Protocol, cast
 
 from litellm.types.integrations.prometheus import (
     UserAPIKeyLabelValues,
-    _sanitize_prometheus_label_name,
-    _sanitize_prometheus_label_value,
+    sanitize_prometheus_label_name,
+    sanitize_prometheus_label_value,
 )
 
 _get_end_user_id_for_cost_tracking = None
 
 
-def _get_cached_end_user_id_for_cost_tracking():
+class _EndUserIdGetter(Protocol):
+    def __call__(
+        self,
+        litellm_params: dict[str, object],
+        service_type: Literal["litellm_logging", "prometheus"] = "litellm_logging",
+    ) -> str | None: ...
+
+
+def get_cached_end_user_id_for_cost_tracking() -> _EndUserIdGetter:
     """
     Get cached get_end_user_id_for_cost_tracking function.
     Lazy imports on first call to avoid loading utils.py at import time (60MB saved).
@@ -31,6 +39,9 @@ def _get_cached_end_user_id_for_cost_tracking():
     return _get_end_user_id_for_cost_tracking
 
 
+_get_cached_end_user_id_for_cost_tracking = get_cached_end_user_id_for_cost_tracking
+
+
 class PrometheusLabelFactoryContext:
     """
     Precomputes per-request label inputs so prometheus_label_factory can subset
@@ -38,11 +49,11 @@ class PrometheusLabelFactoryContext:
     """
 
     __slots__ = (
-        "_custom_by_sanitized_key",
         "_resolved_end_user",
-        "_sanitized_enum",
-        "_tag_labels",
+        "custom_by_sanitized_key",
         "enum_values",
+        "sanitized_enum",
+        "tag_labels",
     )
 
     _END_USER_NOT_COMPUTED = object()
@@ -50,27 +61,51 @@ class PrometheusLabelFactoryContext:
     def __init__(self, enum_values: UserAPIKeyLabelValues) -> None:
         self.enum_values = enum_values
         enum_dict: Final = enum_values.model_dump()
-        self._sanitized_enum: dict[str, str | None] = {
-            k: _sanitize_prometheus_label_value(v) for k, v in enum_dict.items()
+        self.sanitized_enum: dict[str, str | None] = {
+            k: sanitize_prometheus_label_value(v) for k, v in enum_dict.items()
         }
-        self._custom_by_sanitized_key: dict[str, str | None] = {}
+        self.custom_by_sanitized_key: dict[str, str | None] = {}
         if enum_values.custom_metadata_labels is not None:
             for key, value in enum_values.custom_metadata_labels.items():
-                sk = _sanitize_prometheus_label_name(key)
-                self._custom_by_sanitized_key[sk] = _sanitize_prometheus_label_value(value)
-        self._tag_labels: dict[str, str | None] = {}
+                sk = sanitize_prometheus_label_name(key)
+                self.custom_by_sanitized_key[sk] = sanitize_prometheus_label_value(value)
+        self.tag_labels: dict[str, str | None] = {}
         if enum_values.tags is not None:
             # Late import avoids circular import: ``prometheus`` imports this module.
             from litellm.integrations.prometheus import get_custom_labels_from_tags
 
             for k, v in get_custom_labels_from_tags(enum_values.tags).items():
-                self._tag_labels[k] = _sanitize_prometheus_label_value(v)
+                self.tag_labels[k] = sanitize_prometheus_label_value(v)
         # Use a dedicated sentinel so `None` can be cached as a computed result.
         self._resolved_end_user: object = self._END_USER_NOT_COMPUTED
 
+    @property
+    def _custom_by_sanitized_key(self) -> dict[str, str | None]:
+        return self.custom_by_sanitized_key
+
+    @_custom_by_sanitized_key.setter
+    def _custom_by_sanitized_key(self, value: dict[str, str | None]) -> None:
+        self.custom_by_sanitized_key = value
+
+    @property
+    def _sanitized_enum(self) -> dict[str, str | None]:
+        return self.sanitized_enum
+
+    @_sanitized_enum.setter
+    def _sanitized_enum(self, value: dict[str, str | None]) -> None:
+        self.sanitized_enum = value
+
+    @property
+    def _tag_labels(self) -> dict[str, str | None]:
+        return self.tag_labels
+
+    @_tag_labels.setter
+    def _tag_labels(self, value: dict[str, str | None]) -> None:
+        self.tag_labels = value
+
     def get_resolved_end_user(self) -> str | None:
         if self._resolved_end_user is self._END_USER_NOT_COMPUTED:
-            fn: Final = _get_cached_end_user_id_for_cost_tracking()
+            fn: Final = get_cached_end_user_id_for_cost_tracking()
             self._resolved_end_user = fn(
                 litellm_params={"user_api_key_end_user_id": self.enum_values.end_user},
                 service_type="prometheus",

litellm/integrations/prometheus_services.py

@@ -113,7 +113,9 @@ class PrometheusServicesLogger:
         """
         Helper function to get a metric from the registry by name.
         """
-        return self.REGISTRY._names_to_collectors.get(metric_name)
+        return self.REGISTRY._names_to_collectors.get(  # pyright: ignore[reportPrivateUsage]  # Registry lookup has no public API
+            metric_name
+        )
 
     def create_histogram(self, service: str, type_of_request: str):
         metric_name: Final = f"litellm_{service}_{type_of_request}"
@@ -196,7 +198,7 @@ class PrometheusServicesLogger:
                         labels=payload.service.value,
                         amount=payload.duration,
                     )
-                elif isinstance(obj, self.Counter) and "total_requests" in obj._name:
+                elif isinstance(obj, self.Counter) and "total_requests" in obj._name:  # pyright: ignore[reportPrivateUsage]  # Metric names have no public accessor
                     self.increment_counter(
                         counter=obj,
                         labels=payload.service.value,
@@ -233,7 +235,7 @@ class PrometheusServicesLogger:
                         labels=payload.service.value,
                         amount=payload.duration,
                     )
-                elif isinstance(obj, self.Counter) and "total_requests" in obj._name:
+                elif isinstance(obj, self.Counter) and "total_requests" in obj._name:  # pyright: ignore[reportPrivateUsage]  # Metric names have no public accessor
                     self.increment_counter(
                         counter=obj,
                         labels=payload.service.value,
@@ -262,7 +264,7 @@ class PrometheusServicesLogger:
             for obj in prom_objects:
                 # increment both failed and total requests
                 if isinstance(obj, self.Counter):
-                    if "failed_requests" in obj._name:
+                    if "failed_requests" in obj._name:  # pyright: ignore[reportPrivateUsage]  # Metric names have no public accessor
                         self.increment_counter(
                             counter=obj,
                             labels=payload.service.value,

litellm/integrations/weave/weave_otel.py

@@ -106,7 +106,7 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
             safe_set_attribute(span, OpenInferenceSpanAttributes.OUTPUT_VALUE, safe_dumps(output_dict))
 
 
-def _get_weave_authorization_header(api_key: str) -> str:
+def get_weave_authorization_header(api_key: str) -> str:
     """
     Get the authorization header for Weave OpenTelemetry.
 
@@ -117,6 +117,9 @@ def _get_weave_authorization_header(api_key: str) -> str:
     return f"Basic {auth_header}"
 
 
+_get_weave_authorization_header = get_weave_authorization_header
+
+
 def weave_otel_endpoint(host: str | None) -> str:
     """The OTLP traces endpoint for a self-managed ``host``, else Weave cloud."""
     if not host:
@@ -155,7 +158,7 @@ def get_weave_otel_config() -> WeaveOtelConfig:
     verbose_logger.debug("Using Weave OTEL endpoint: %s", endpoint)
 
     # Weave uses Basic auth with format: api:<WANDB_API_KEY>
-    auth_header: Final = _get_weave_authorization_header(api_key=api_key)
+    auth_header: Final = get_weave_authorization_header(api_key=api_key)
     otlp_auth_headers: Final = f"Authorization={auth_header},project_id={project_id}"
 
     # Set standard OTEL environment variables
@@ -320,7 +323,7 @@ class WeaveOtelLogger(OpenTelemetry):
         dynamic_weave_project_id: Final = standard_callback_dynamic_params.get("weave_project_id")
 
         if dynamic_wandb_api_key:
-            auth_header: Final = _get_weave_authorization_header(
+            auth_header: Final = get_weave_authorization_header(
                 api_key=dynamic_wandb_api_key,
             )
             dynamic_headers["Authorization"] = auth_header

litellm/interactions/streaming_iterator.py

@@ -72,7 +72,7 @@ class BaseInteractionsAPIStreamingIterator:
             return None
 
         # Handle SSE format (data: {...})
-        stripped_chunk: Final = CustomStreamWrapper._strip_sse_data_from_chunk(chunk)
+        stripped_chunk: Final = CustomStreamWrapper.strip_sse_data_from_chunk(chunk)
         if stripped_chunk is None:
             return None
 

litellm/litellm_core_utils/core_helpers.py

@@ -5,7 +5,7 @@ import logging
 import re
 from collections.abc import Collection, Iterable, Mapping
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
 
 import httpx
 from pydantic import TypeAdapter, ValidationError
@@ -427,30 +427,43 @@ def reconstruct_model_name(
 
 
 # Helper functions used for OTEL logging
-def _get_parent_otel_span_from_kwargs(
-    kwargs: dict | None = None,
+def get_parent_otel_span_from_kwargs(
+    kwargs: dict[str, object] | None = None,
 ) -> Span | None:
     try:
         if kwargs is None:
             return None
         litellm_params: Final = kwargs.get("litellm_params")
-        _metadata: Final = kwargs.get("metadata") or {}
-        if "litellm_parent_otel_span" in _metadata:
-            return _metadata["litellm_parent_otel_span"]
+        metadata: Final = cast(  # cast-ok: metadata is caller-provided request data
+            Mapping[str, object], kwargs.get("metadata") or {}
+        )
+        if "litellm_parent_otel_span" in metadata:
+            return cast(  # cast-ok: tracing metadata crosses an external boundary
+                Span | None, metadata["litellm_parent_otel_span"]
+            )
         elif (
             litellm_params is not None
-            and litellm_params.get("metadata") is not None
-            and "litellm_parent_otel_span" in litellm_params.get("metadata", {})
+            and cast(Mapping[str, object], litellm_params).get("metadata") is not None
+            and "litellm_parent_otel_span"
+            in cast(
+                Mapping[str, object],
+                cast(Mapping[str, object], litellm_params).get("metadata", {}),
+            )
         ):
-            return litellm_params["metadata"]["litellm_parent_otel_span"]
+            typed_litellm_params: Final = cast(Mapping[str, object], litellm_params)
+            litellm_metadata: Final = cast(Mapping[str, object], typed_litellm_params["metadata"])
+            return cast(Span | None, litellm_metadata["litellm_parent_otel_span"])
         elif "litellm_parent_otel_span" in kwargs:
-            return kwargs["litellm_parent_otel_span"]
+            return cast(Span | None, kwargs["litellm_parent_otel_span"])
         return None
     except Exception as e:
         verbose_logger.exception("Error in _get_parent_otel_span_from_kwargs: " + str(e))
         return None
 
 
+_get_parent_otel_span_from_kwargs = get_parent_otel_span_from_kwargs
+
+
 def process_response_headers(
     response_headers: httpx.Headers | dict,
     preserve_litellm_internal_headers: bool = False,
@@ -760,12 +773,18 @@ _NO_HEADERS: Final[Mapping[str, object]] = MappingProxyType({})
 class _CarriesHiddenParams(Protocol):
     _hidden_params: dict[str, object]  # mutable-ok: the responses billed here keep hidden params in a plain dict
 
+    @property
+    def hidden_params(self) -> dict[str, object]: ...  # mutable-ok: API requires mutation
+
+    @hidden_params.setter
+    def hidden_params(self, hidden_params: dict[str, object]) -> None: ...  # mutable-ok: API requires mutation
+
 
 def set_response_cost_in_hidden_params(response: _CarriesHiddenParams, cost: float | None) -> None:
     """Record a provider-reported cost where the cost calculator looks before the price map."""
     if cost is None:
         return
-    hidden_params: Final = response._hidden_params  # pyright: ignore[reportPrivateUsage]  # no public accessor
+    hidden_params: Final = response.hidden_params
     additional_headers: Final[object] = hidden_params.get("additional_headers")
     merged: Final[dict[str, object]] = {  # mutable-ok: assigned into the plain-dict hidden params
         **(additional_headers if isinstance(additional_headers, Mapping) else _NO_HEADERS),
@@ -781,7 +800,7 @@ _PROVIDER_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
 def set_provider_response_headers_in_hidden_params(
     response: _CarriesHiddenParams, headers: httpx.Headers | Mapping[str, str]
 ) -> None:
-    hidden_params: Final = response._hidden_params  # pyright: ignore[reportPrivateUsage]  # no public accessor
+    hidden_params: Final = response.hidden_params
     existing_additional_headers: Final[object] = hidden_params.get("additional_headers")
     raw_headers: Final[dict[str, str]] = dict(headers)  # mutable-ok: stored as the plain-dict hidden param
     additional_headers: Final[dict[str, object]] = {  # mutable-ok: assigned into the plain-dict hidden params

litellm/litellm_core_utils/dd_tracing.py

@@ -57,11 +57,14 @@ def _should_use_dd_tracer():
     return get_secret_bool("USE_DDTRACE", False) is True
 
 
-def _should_use_dd_profiler():
+def should_use_dd_profiler() -> bool:
     """Returns True if `USE_DDPROFILER` is set to True in .env"""
     return get_secret_bool("USE_DDPROFILER", False) is True
 
 
+_should_use_dd_profiler = should_use_dd_profiler
+
+
 # Initialize tracer
 should_use_dd_tracer: Final = _should_use_dd_tracer()
 tracer: NullTracer | DD_TRACER = NullTracer()

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -9,13 +9,13 @@ from typing import Final, Protocol, cast
 import httpx
 
 import litellm
-from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
+from litellm._logging import _ENABLE_SECRET_REDACTION, redact_string, verbose_logger
 from litellm.litellm_core_utils.bug_report import (
     bug_report_notice,
     build_bug_report,
     should_report_bug,
 )
-from litellm.litellm_core_utils.secret_redaction import redact_string
+from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
 from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
@@ -28,6 +28,7 @@ from ..exceptions import (
     ContextWindowExceededError,
     InternalServerError,
     NotFoundError,
+    PaymentRequiredError,
     PermissionDeniedError,
     RateLimitError,
     ServiceUnavailableError,
@@ -2128,7 +2129,7 @@ def _map_azure_exception(
     else:
         # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
         raise APIConnectionError(
-            message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}",
+            message=f"{exception_provider} APIConnectionError - {message}\n{redact_string(traceback.format_exc())}",
             llm_provider="azure",
             model=model,
             litellm_debug_info=extra_information,
@@ -2327,6 +2328,14 @@ def _map_exception_by_status(
                 litellm_debug_info=extra_information,
                 exception_status_code=status_code,
             )
+        case 402:
+            raise PaymentRequiredError(
+                message=message,
+                model=model,
+                llm_provider=custom_llm_provider,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
         case _ if status_code < 500:
             raise BadRequestError(
                 message=message,
@@ -2373,18 +2382,20 @@ def exception_type(
             "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"
         )
         print(  # noqa: T201
-            "LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'."
+            "LiteLLM.Info: If you need to debug this error, use `litellm.turn_on_debug()'."
         )
         print()  # noqa: T201
 
     litellm_response_headers: Final = _get_response_headers(original_exception=original_exception)
     try:
-        error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
+        error_str = (
+            redact_secret_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
+        )
         extra_information = ""
         if model or custom_llm_provider:
             if hasattr(original_exception, "message"):
                 error_str = (
-                    redact_string(str(original_exception.message))
+                    redact_secret_string(str(original_exception.message))
                     if _ENABLE_SECRET_REDACTION
                     else str(original_exception.message)
                 )
@@ -2425,7 +2436,7 @@ def exception_type(
                     extra_information += f"\nvertex_location: `{_vertex_location}`\n"
 
                 # on litellm proxy add key name + team to exceptions
-                extra_information = _add_key_name_and_team_to_alert(request_info=extra_information, metadata=_metadata)
+                extra_information = add_key_name_and_team_to_alert(request_info=extra_information, metadata=_metadata)
             except Exception:
                 # DO NOT LET this Block raising the original exception
                 pass
@@ -2687,7 +2698,7 @@ def exception_type(
             else:
                 raise APIConnectionError(
                     message=(
-                        f"{original_exception}\n{_redact_string(traceback.format_exc())}"
+                        f"{original_exception}\n{redact_string(traceback.format_exc())}"
                         + (
                             "\n"
                             + bug_report_notice(
@@ -2726,7 +2737,7 @@ def exception_type(
                     setattr(e, "litellm_response_headers", litellm_response_headers)
                     raise e  # it's already mapped
             raised_exc: Final = APIConnectionError(
-                message=f"{original_exception}\n{_redact_string(traceback.format_exc())}",
+                message=f"{original_exception}\n{redact_string(traceback.format_exc())}",
                 llm_provider="",
                 model="",
             )
@@ -2766,7 +2777,7 @@ def exception_logging(
         )
 
 
-def _add_key_name_and_team_to_alert(request_info: str, metadata: dict) -> str:
+def add_key_name_and_team_to_alert(request_info: str, metadata: Mapping[str, object]) -> str:
     """
     Internal helper function for litellm proxy
     Add the Key Name + Team Name to the error
@@ -2783,3 +2794,6 @@ def _add_key_name_and_team_to_alert(request_info: str, metadata: dict) -> str:
         return request_info
     except Exception:
         return request_info
+
+
+_add_key_name_and_team_to_alert = add_key_name_and_team_to_alert

litellm/litellm_core_utils/get_litellm_params.py

@@ -2,7 +2,7 @@ import reprlib
 from collections.abc import Mapping, MutableMapping
 from dataclasses import dataclass, fields
 from types import MappingProxyType
-from typing import Final
+from typing import Final, cast
 
 from pydantic import TypeAdapter, ValidationError
 
@@ -123,17 +123,25 @@ def with_control_options(litellm_params: Mapping[str, object], control: ControlO
     return {**litellm_params, CONTROL_OPTIONS_KEY: control}
 
 
-def _get_base_model_from_litellm_call_metadata(
-    metadata: dict | None,
+def get_base_model_from_litellm_call_metadata(
+    metadata: Mapping[str, object] | None,
 ) -> str | None:
     if metadata is None:
         return None
     model_info: Final = metadata.get("model_info")
     if model_info:
-        return model_info.get("base_model")
+        model_info_mapping: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
+            Mapping[str, object], model_info
+        )
+        return cast(  # cast-ok: model metadata values are caller-provided
+            str | None, model_info_mapping.get("base_model")
+        )
     return None
 
 
+_get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata
+
+
 def get_litellm_params(
     api_key: str | None = None,
     force_timeout=600,
@@ -229,7 +237,7 @@ def get_litellm_params(
         "azure_ad_token_provider": azure_ad_token_provider,
         "user_continue_message": user_continue_message,
         "base_model": base_model
-        or (_get_base_model_from_litellm_call_metadata(metadata=metadata) if metadata else None),
+        or (get_base_model_from_litellm_call_metadata(metadata=metadata) if metadata else None),
         "litellm_trace_id": litellm_trace_id,
         "litellm_session_id": litellm_session_id,
         "hf_model_name": hf_model_name,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -53,7 +53,7 @@ def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool:
     return url_path == endpoint_path or url_path.startswith(endpoint_path + "/")
 
 
-def _is_non_openai_azure_model(model: str) -> bool:
+def is_non_openai_azure_model(model: str) -> bool:
     try:
         model_name: Final = model.split("/", 1)[1]
         if model_name in litellm.cohere_chat_models or f"mistral/{model_name}" in litellm.mistral_chat_models:
@@ -63,6 +63,9 @@ def _is_non_openai_azure_model(model: str) -> bool:
     return False
 
 
+_is_non_openai_azure_model = is_non_openai_azure_model
+
+
 def _is_azure_claude_model(model: str) -> bool:
     """
     Check if a model name contains 'claude' (case-insensitive).
@@ -195,7 +198,7 @@ def get_llm_provider(
         # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere
         # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus
         if model.split("/", 1)[0] == "azure":
-            if _is_non_openai_azure_model(model):
+            if is_non_openai_azure_model(model):
                 custom_llm_provider = "openai"
                 return model, custom_llm_provider, dynamic_api_key, api_base
 

litellm/litellm_core_utils/get_model_cost_map.py

@@ -105,13 +105,15 @@ class GetModelCostMap:
         cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()})
 
     @classmethod
-    def _get_backup_model_count(cls) -> int:
+    def get_backup_model_count(cls) -> int:
         """Return the number of models in the local backup (cached int)."""
         if cls._backup_model_count < 0:
             backup: Final = cls.load_local_model_cost_map()
             cls._backup_model_count = _count_model_entries(backup)
         return cls._backup_model_count
 
+    _get_backup_model_count = get_backup_model_count
+
     @staticmethod
     def _check_is_valid_dict(fetched_map: dict) -> bool:
         """Check 1: fetched map is a non-empty dict."""
@@ -402,7 +404,7 @@ async def refetch_model_cost_map(
         return result
     if not GetModelCostMap.validate_model_cost_map(
         fetched_map=result.model_cost_map,
-        backup_model_count=GetModelCostMap._get_backup_model_count(),
+        backup_model_count=GetModelCostMap.get_backup_model_count(),
     ):
         return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation")
     _cost_map_source_info.loaded_at = datetime.now(timezone.utc)
@@ -600,7 +602,7 @@ def _retry_remote_fetch_in_background(
         _litellm_import_complete.wait()
         if not GetModelCostMap.validate_model_cost_map(
             fetched_map=result.model_cost_map,
-            backup_model_count=GetModelCostMap._get_backup_model_count(),  # pyright: ignore[reportPrivateUsage]  # integrity cache
+            backup_model_count=GetModelCostMap.get_backup_model_count(),
         ):
             verbose_logger.warning(
                 "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s",
@@ -683,7 +685,7 @@ def get_model_cost_map(
     # Validate using cached count (cheap int comparison, no file I/O)
     if not GetModelCostMap.validate_model_cost_map(
         fetched_map=content,
-        backup_model_count=GetModelCostMap._get_backup_model_count(),
+        backup_model_count=GetModelCostMap.get_backup_model_count(),
     ):
         verbose_logger.warning(
             "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s",

litellm/litellm_core_utils/health_check_helpers.py

@@ -96,7 +96,7 @@ class HealthCheckHelpers:
         return {}
 
     @staticmethod
-    def _update_model_params_with_health_check_tracking_information(
+    def update_model_params_with_health_check_tracking_information(
         model_params: dict,
     ) -> dict:
         """
@@ -120,6 +120,10 @@ class HealthCheckHelpers:
         )
         return model_params
 
+    _update_model_params_with_health_check_tracking_information = (
+        update_model_params_with_health_check_tracking_information
+    )
+
     @staticmethod
     def _get_metadata_for_health_check_call():
         """
@@ -219,7 +223,7 @@ class HealthCheckHelpers:
             get_audio_file_for_health_check,
         )
         from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params
-        from litellm.realtime_api.main import _realtime_health_check
+        from litellm.realtime_api.main import realtime_health_check
 
         return {
             "chat": lambda: litellm.acompletion(
@@ -264,7 +268,7 @@ class HealthCheckHelpers:
                 query=prompt or "",
                 documents=["my sample text"],
             ),
-            "realtime": lambda: _realtime_health_check(
+            "realtime": lambda: realtime_health_check(
                 model=model,
                 custom_llm_provider=custom_llm_provider,
                 api_base=model_params.get("api_base", None),

litellm/litellm_core_utils/health_check_utils.py

@@ -2,6 +2,7 @@
 Utils used for litellm.ahealth_check()
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 from pydantic import TypeAdapter
@@ -17,8 +18,8 @@ def _filter_model_params(model_params: dict) -> dict:
     return {k: v for k, v in model_params.items() if k != "messages"}
 
 
-def _create_health_check_response(response_headers: dict) -> dict:
-    response: Final = {}
+def create_health_check_response(response_headers: Mapping[str, object]) -> dict[str, object]:
+    response: Final[dict[str, object]] = {}
 
     if response_headers.get("x-ratelimit-remaining-requests", None) is not None:  # not provided for dall-e requests
         response["x-ratelimit-remaining-requests"] = response_headers["x-ratelimit-remaining-requests"]
@@ -29,3 +30,6 @@ def _create_health_check_response(response_headers: dict) -> dict:
     if response_headers.get("x-ms-region", None) is not None:
         response["x-ms-region"] = response_headers["x-ms-region"]
     return response
+
+
+_create_health_check_response = create_health_check_response

litellm/litellm_core_utils/hidden_params.py

@@ -0,0 +1,114 @@
+from collections.abc import Iterator, MutableMapping
+from typing import Final, Protocol, cast
+
+from litellm.types.llms.base import HiddenParams
+
+_HIDDEN_PARAMS_ATTR: Final = "_hidden_params"
+HIDDEN_PARAMS_ATTR: Final = _HIDDEN_PARAMS_ATTR
+
+
+class _SupportsItemAssignment(Protocol):
+    def __setitem__(self, key: str, value: object) -> None: ...
+
+
+class HiddenParamsModelView(MutableMapping[str, object]):
+    def __init__(self, hidden_params: HiddenParams) -> None:
+        self._hidden_params = hidden_params
+
+    def __getitem__(self, key: str) -> object:
+        fields: Final = self._keys()
+        if key not in fields:
+            raise KeyError(key)
+        try:
+            return cast(object, getattr(self._hidden_params, key))
+        except AttributeError as error:
+            raise KeyError(key) from error
+
+    def __setitem__(self, key: str, value: object) -> None:
+        setattr(self._hidden_params, key, value)
+
+    def __delitem__(self, key: str) -> None:
+        fields: Final = self._keys()
+        if key not in fields:
+            raise KeyError(key)
+        try:
+            delattr(self._hidden_params, key)
+        except AttributeError as error:
+            raise KeyError(key) from error
+        self._hidden_params.model_fields_set.discard(key)
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._keys())
+
+    def __len__(self) -> int:
+        return len(self._keys())
+
+    def _keys(self) -> frozenset[str]:
+        return frozenset(self._hidden_params.model_fields_set) | frozenset(self._hidden_params.model_extra or {})
+
+
+def _get_hidden_params_storage(obj: object) -> object | None:
+    return obj.get(_HIDDEN_PARAMS_ATTR) if isinstance(obj, dict) else getattr(obj, _HIDDEN_PARAMS_ATTR, None)
+
+
+def get_hidden_params_storage(obj: object) -> dict[str, object] | HiddenParams | None:
+    hidden_params: Final[object | None] = _get_hidden_params_storage(obj)
+    if isinstance(hidden_params, dict):
+        return cast(  # cast-ok: runtime dict validation preserves dynamically typed legacy storage
+            dict[str, object], hidden_params
+        )
+    if isinstance(hidden_params, HiddenParams):
+        return hidden_params
+    return None
+
+
+def _as_hidden_params_mapping(hidden_params: object | None) -> MutableMapping[str, object] | None:
+    if isinstance(hidden_params, dict):
+        return cast(  # cast-ok: runtime dict validation preserves dynamically typed legacy storage
+            dict[str, object], hidden_params
+        )
+    if isinstance(hidden_params, HiddenParams):
+        return HiddenParamsModelView(hidden_params)
+    return None
+
+
+def get_hidden_params(obj: object) -> MutableMapping[str, object] | None:
+    hidden_params: Final[object | None] = _get_hidden_params_storage(obj)
+    return _as_hidden_params_mapping(hidden_params)
+
+
+def set_hidden_params(obj: object, hidden_params: dict[str, object] | HiddenParams) -> None:
+    if isinstance(obj, dict):
+        obj[_HIDDEN_PARAMS_ATTR] = hidden_params
+    else:
+        setattr(obj, _HIDDEN_PARAMS_ATTR, hidden_params)
+
+
+def set_hidden_param(obj: object, key: str, value: object) -> None:
+    hidden_params: Final[object | None] = _get_hidden_params_storage(obj)
+    if hidden_params is None:
+        set_hidden_params(obj, {key: value})
+        return
+    if isinstance(hidden_params, dict):
+        cast(  # cast-ok: runtime dict validation preserves dynamically typed legacy storage
+            dict[str, object], hidden_params
+        )[key] = value
+        return
+    if hasattr(hidden_params, "__setitem__"):
+        cast(  # cast-ok: hasattr validates the legacy storage assignment protocol
+            _SupportsItemAssignment, hidden_params
+        )[key] = value
+        return
+    raise TypeError(f"unsupported hidden params storage: {type(hidden_params).__name__}")
+
+
+def get_or_create_hidden_params(obj: object) -> MutableMapping[str, object]:
+    hidden_params: Final[object | None] = _get_hidden_params_storage(obj)
+    if hidden_params is None:
+        created_hidden_params: Final[dict[str, object]] = {}
+        set_hidden_params(obj, created_hidden_params)
+        return created_hidden_params
+    hidden_params_mapping: Final[MutableMapping[str, object] | None] = _as_hidden_params_mapping(hidden_params)
+    if hidden_params_mapping is not None:
+        return hidden_params_mapping
+    raise TypeError(f"unsupported hidden params storage: {type(hidden_params).__name__}")

litellm/litellm_core_utils/litellm_logging.py

@@ -11,11 +11,11 @@ import subprocess
 import sys
 import time
 import traceback
-from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
 from datetime import datetime as dt_object
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
 
 from httpx import Response
 from pydantic import BaseModel, JsonValue
@@ -24,8 +24,8 @@ import litellm
 from litellm import _custom_logger_compatible_callbacks_literal
 from litellm._internal_context import post_response_phase
 from litellm._logging import (
-    _is_debugging_on,
-    _redact_string,
+    is_debugging_on,
+    redact_string,
     session_id_var,
     set_session_id,
     set_trace_id,
@@ -33,7 +33,7 @@ from litellm._logging import (
     verbose_logger,
 )
 from litellm._uuid import uuid
-from litellm.batches.batch_utils import _handle_completed_batch, batch_cost_is_final
+from litellm.batches.batch_utils import batch_cost_is_final, handle_completed_batch
 from litellm.caching.caching import DualCache
 from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.caching.redis_batch import flush_post_call_redis_batches
@@ -47,9 +47,9 @@ from litellm.constants import (
 from litellm.cost_calculator import (
     RealtimeAPITokenUsageProcessor,
     ResponsesWebSocketTokenUsageProcessor,
-    _select_model_name_for_cost_calc,
     get_usage_object,
     pricing_entry_for_cost_calc,
+    select_model_name_for_cost_calc,
 )
 from litellm.exceptions import (
     BudgetExceededError,
@@ -79,6 +79,7 @@ from litellm.litellm_core_utils.core_helpers import (
 )
 from litellm.litellm_core_utils.error_normalization import normalize_error
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
+from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_param
 from litellm.litellm_core_utils.internal_call_metadata import (
     MODEL_ACCESS_GROUP_METADATA_KEY,
     is_unbilled_non_inference_call,
@@ -178,7 +179,7 @@ from litellm.types.utils import (
     Usage,
 )
 from litellm.types.videos.main import VideoObject
-from litellm.utils import _get_base_model_from_metadata, print_verbose
+from litellm.utils import get_base_model_from_metadata, print_verbose
 
 from ..integrations.argilla import ArgillaLogger
 from ..integrations.arize.arize_phoenix import ArizePhoenixLogger
@@ -575,6 +576,38 @@ class Logging(LiteLLMLoggingBaseClass):
     baseline_cache_context: "BaselineCacheContext | None" = None
     baseline_observation: "CapturedBaselineObservation | None" = None
 
+    @property
+    def _defer_async_logging(self) -> bool:
+        return self.defer_async_logging
+
+    @_defer_async_logging.setter
+    def _defer_async_logging(self, value: bool) -> None:
+        self.defer_async_logging = value
+
+    @property
+    def _enqueue_deferred_logging(self) -> Callable[[], None] | None:
+        return self.enqueue_deferred_logging
+
+    @_enqueue_deferred_logging.setter
+    def _enqueue_deferred_logging(self, value: Callable[[], None] | None) -> None:
+        self.enqueue_deferred_logging = value
+
+    @property
+    def _llm_caching_handler(self) -> LLMCachingHandler | None:
+        return self.llm_caching_handler
+
+    @_llm_caching_handler.setter
+    def _llm_caching_handler(self, value: LLMCachingHandler | None) -> None:
+        self.llm_caching_handler = value
+
+    @property
+    def _on_detached_stream_failure(self) -> Callable[[Exception], Awaitable[None]] | None:
+        return self.on_detached_stream_failure
+
+    @_on_detached_stream_failure.setter
+    def _on_detached_stream_failure(self, value: Callable[[Exception], Awaitable[None]] | None) -> None:
+        self.on_detached_stream_failure = value
+
     def __init__(
         self,
         model: str,
@@ -686,7 +719,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # once that response is priced, the same way a non-streamed response is logged
         self.client_facing_stream_model: str | None = None
         self.zero_cost_warned: bool = False
-        self._llm_caching_handler: LLMCachingHandler | None = None
+        self.llm_caching_handler: LLMCachingHandler | None = None
 
         # INITIAL LITELLM_PARAMS
         litellm_params = {}
@@ -721,9 +754,10 @@ class Logging(LiteLLMLoggingBaseClass):
         # Set by proxy request handlers to defer spend-log fire until after
         # post_call guardrails have run; the @client decorator then stores the
         # enqueue closure here instead of firing it immediately.
-        self._defer_async_logging: bool = False
-        self._enqueue_deferred_logging: Callable[[], None] | None = None
-        self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
+        self.defer_async_logging: bool = False
+        self.enqueue_deferred_logging: Callable[[], None] | None = None
+        self._async_success_scheduled: bool = False
+        self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
     def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
@@ -825,7 +859,7 @@ class Logging(LiteLLMLoggingBaseClass):
         trusted-vars channel is the only way credentials reach a per-team logger.
         """
         _trusted_var_prefix: Final = "dd_" if callback == "datadog" else "newrelic_" if callback == "newrelic" else None
-        _custom_logger_init_args: Final[dict | None] = (
+        _custom_logger_init_args: Final[dict[str, object] | None] = (
             {k: v for k, v in self._trusted_callback_vars if k.startswith(_trusted_var_prefix)}
             if _trusted_var_prefix is not None
             else None
@@ -868,8 +902,8 @@ class Logging(LiteLLMLoggingBaseClass):
         checks if web_search_options in kwargs or tools and sets the corresponding attribute in StandardBuiltInToolsParams
         """
         return StandardBuiltInToolsParams(
-            web_search_options=StandardBuiltInToolCostTracking._get_web_search_options(kwargs or {}),
-            file_search=StandardBuiltInToolCostTracking._get_file_search_tool_call(kwargs or {}),
+            web_search_options=StandardBuiltInToolCostTracking.get_web_search_options(kwargs or {}),
+            file_search=StandardBuiltInToolCostTracking.get_file_search_tool_call(kwargs or {}),
         )
 
     def get_router_model_id(self) -> str | None:
@@ -929,7 +963,7 @@ class Logging(LiteLLMLoggingBaseClass):
         }
         self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
         self.logger_fn = litellm_params.get("logger_fn", None)
-        if _is_debugging_on() or self.litellm_request_debug:
+        if is_debugging_on() or self.litellm_request_debug:
             verbose_logger.debug("self.optional_params: %s", self.optional_params)
 
         self.model_call_details.update(
@@ -1392,12 +1426,12 @@ class Logging(LiteLLMLoggingBaseClass):
                             additional_args=additional_args,
                             data=additional_args.get("complete_input_dict", {}),
                         )
-                        _metadata["raw_request"] = _redact_string(str(curl_command))
+                        _metadata["raw_request"] = redact_string(str(curl_command))
                 except Exception as e:
                     self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict(
                         error=str(e),
                     )
-                    _metadata["raw_request"] = _redact_string(
+                    _metadata["raw_request"] = redact_string(
                         f"Unable to Log \
                         raw request: {e}"
                     )
@@ -1494,7 +1528,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         Prints the RAW curl command sent from LiteLLM
         """
-        if _is_debugging_on() or self.litellm_request_debug:
+        if is_debugging_on() or self.litellm_request_debug:
             if litellm.json_logs:
                 masked_headers: Final = self._get_masked_headers(headers or {})
                 masked_api_base: Final = self._get_masked_api_base(str(api_base or ""))
@@ -1554,7 +1588,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         Masks the headers of the request sent from LiteLLM
         """
-        return _get_masked_values(headers, ignore_sensitive_values=ignore_sensitive_headers)
+        return get_masked_values(headers, ignore_sensitive_values=ignore_sensitive_headers)
 
     def post_call(self, original_response, input=None, api_key=None, additional_args={}):
         # Log the exact result from the LLM API, for streaming - log the type of response received
@@ -1799,29 +1833,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if margin_total_amount is not None:
             self.cost_breakdown["margin_total_amount"] = margin_total_amount
 
-    def _response_cost_calculator(
+    def response_cost_calculator(
         self,
-        result: Union[
-            ModelResponse,
-            ModelResponseStream,
-            EmbeddingResponse,
-            ImageResponse,
-            TranscriptionResponse,
-            TextCompletionResponse,
-            HttpxBinaryResponseContent,
-            RerankResponse,
-            Batch,
-            FineTuningJob,
-            ResponsesAPIResponse,
-            ResponseCompletedEvent,
-            OpenAIFileObject,
-            LiteLLMRealtimeStreamLoggingObject,
-            OpenAIModerationResponse,
-            "SearchResponse",
-            DecisionsResponse,
-            dict,
-            list,
-        ],
+        result: object,
         cache_hit: bool | None = None,
         litellm_model_name: str | None = None,
         router_model_id: str | None = None,
@@ -1844,16 +1858,15 @@ class Logging(LiteLLMLoggingBaseClass):
             return 0.0
 
         transformed_result: Final = self._generate_content_result_as_model_response(result)
-        if transformed_result is not None:
-            result = transformed_result
+        response_result: Final[object] = transformed_result if transformed_result is not None else result
 
         priced_result: Final = (
-            result.response
-            if isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent))
-            else result
+            response_result.response
+            if isinstance(response_result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent))
+            else response_result
         )
 
-        result_hidden_params: Final = getattr(priced_result, "_hidden_params", None) or MappingProxyType({})
+        result_hidden_params: Final = getattr(priced_result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
         if isinstance(priced_result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(
             priced_result, "_hidden_params"
         ):
@@ -1880,18 +1893,28 @@ class Logging(LiteLLMLoggingBaseClass):
         ## RESPONSE COST ##
         custom_pricing: Final = self._custom_pricing_for(priced_result)
 
-        prompt = self._prompt_for_cost_calculation()
+        prompt: Final = self._prompt_for_cost_calculation()
 
-        if cache_hit is None:
-            cache_hit = self.model_call_details.get("cache_hit", False)
+        model_value: Final = litellm_model_name or self.model
+        cost_cache_hit_value: Final = (
+            self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
+        )
+        cost_cache_hit: Final[bool | None] = cast(  # cast-ok: callback metadata is caller-provided
+            bool | None, cost_cache_hit_value
+        )
+        provider_value: Final = self.model_call_details.get("custom_llm_provider", None)
+        cost_custom_llm_provider: Final[str | None] = cast(  # cast-ok: callback metadata is caller-provided
+            str | None, provider_value
+        )
+        base_model: Final = get_base_model_from_metadata(model_call_details=self.model_call_details)
 
         try:
             response_cost_calculator_kwargs: Final = {
                 "response_object": priced_result,
-                "model": litellm_model_name or self.model,
-                "cache_hit": cache_hit,
-                "custom_llm_provider": self.model_call_details.get("custom_llm_provider", None),
-                "base_model": _get_base_model_from_metadata(model_call_details=self.model_call_details),
+                "model": model_value,
+                "cache_hit": cost_cache_hit,
+                "custom_llm_provider": cost_custom_llm_provider,
+                "base_model": base_model,
                 "call_type": self.call_type,
                 "optional_params": self.optional_params,
                 "custom_pricing": custom_pricing,
@@ -1905,13 +1928,13 @@ class Logging(LiteLLMLoggingBaseClass):
                     else None
                 ),
                 "vertex_location": _resolve_vertex_location_for_cost(
-                    custom_llm_provider=self.model_call_details.get("custom_llm_provider", None),
+                    custom_llm_provider=cost_custom_llm_provider,
                     litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None),
                     optional_params=self.optional_params,
-                    model=litellm_model_name or self.model,
+                    model=model_value,
                 ),
                 "region_name": _resolve_mantle_region_for_cost(
-                    custom_llm_provider=self.model_call_details.get("custom_llm_provider", None),
+                    custom_llm_provider=cost_custom_llm_provider,
                     litellm_params=self.model_call_details.get("litellm_params"),
                 ),
             }
@@ -1945,12 +1968,12 @@ class Logging(LiteLLMLoggingBaseClass):
             debug_info = StandardLoggingModelCostFailureDebugInformation(
                 error_str=str(e),
                 traceback_str=_get_traceback_str_for_error(str(e)),
-                model=response_cost_calculator_kwargs["model"],
-                cache_hit=response_cost_calculator_kwargs["cache_hit"],
-                custom_llm_provider=response_cost_calculator_kwargs["custom_llm_provider"],
-                base_model=response_cost_calculator_kwargs["base_model"],
-                call_type=response_cost_calculator_kwargs["call_type"],
-                custom_pricing=response_cost_calculator_kwargs["custom_pricing"],
+                model=model_value,
+                cache_hit=cost_cache_hit,
+                custom_llm_provider=cost_custom_llm_provider,
+                base_model=base_model,
+                call_type=self.call_type,
+                custom_pricing=custom_pricing,
             )
             verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info)
             self.model_call_details["response_cost_failure_debug_information"] = debug_info
@@ -1964,6 +1987,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         return None
 
+    _response_cost_calculator = response_cost_calculator
+
     def _record_zero_cost_diagnostic(
         self,
         result: object,
@@ -2017,7 +2042,7 @@ class Logging(LiteLLMLoggingBaseClass):
             completion_response=result,
             custom_llm_provider=custom_llm_provider,
             custom_pricing=self._custom_pricing_for(result),
-            base_model=_get_base_model_from_metadata(model_call_details=self.model_call_details),
+            base_model=get_base_model_from_metadata(model_call_details=self.model_call_details),
             router_model_id=router_model_id or self.get_router_model_id(),
             region_name=_resolve_mantle_region_for_cost(
                 custom_llm_provider=custom_llm_provider,
@@ -2043,7 +2068,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def _custom_pricing_for(self, result: object) -> bool:
         litellm_params: Final = getattr(self, "litellm_params", None)
-        result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({})
+        result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
         additional_headers: Final = (
             result_hidden_params.get("additional_headers")
             if isinstance(result_hidden_params, dict)
@@ -2116,10 +2141,10 @@ class Logging(LiteLLMLoggingBaseClass):
         | FineTuningJob,
         cache_hit: bool | None = None,
     ) -> float | None:
-        return self._response_cost_calculator(result=result, cache_hit=cache_hit)
+        return self.response_cost_calculator(result=result, cache_hit=cache_hit)
 
     @staticmethod
-    def _is_sync_litellm_request(litellm_params: dict) -> bool:
+    def is_sync_litellm_request(litellm_params: Mapping[str, object]) -> bool:
         """True for sync SDK entrypoints (``completion``), false for async (``acompletion``, etc.)."""
         return (
             litellm_params.get(CallTypes.acompletion.value, False) is not True
@@ -2134,6 +2159,8 @@ class Logging(LiteLLMLoggingBaseClass):
             and litellm_params.get(CallTypes.arealtime.value, False) is not True
         )
 
+    _is_sync_litellm_request = is_sync_litellm_request
+
     def _is_assembled_stream_success(self, result=None) -> bool:
         """Final assembled stream export (not a per-chunk success call).
 
@@ -2175,7 +2202,7 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details["has_dispatched_final_stream_success"] = True
 
         litellm_params: Final = self.model_call_details.get("litellm_params", {}) or {}
-        sync_sdk: Final = self._is_sync_litellm_request(litellm_params)
+        sync_sdk: Final = self.is_sync_litellm_request(litellm_params)
         passthrough: Final = self.call_type == CallTypes.pass_through.value
         if sync_sdk and not prefer_async_handlers and not passthrough:
             self.success_handler(
@@ -2216,7 +2243,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """Bill a fully streamed response on the failure log when a post-call hook rejects it."""
         usage: Final = getattr(assembled, "usage", None)
         if isinstance(usage, Usage):
-            self.record_partial_usage_for_failure(usage, self._response_cost_calculator(result=assembled) or 0.0)
+            self.record_partial_usage_for_failure(usage, self.response_cost_calculator(result=assembled) or 0.0)
 
     async def dispatch_failure_handlers(
         self,
@@ -2236,7 +2263,7 @@ class Logging(LiteLLMLoggingBaseClass):
         the request failed).
         """
         litellm_params: Final = self.model_call_details.get("litellm_params", {}) or {}
-        sync_sdk: Final = self._is_sync_litellm_request(litellm_params)
+        sync_sdk: Final = self.is_sync_litellm_request(litellm_params)
         passthrough: Final = self.call_type == CallTypes.pass_through.value
         if sync_sdk and not prefer_async_handlers and not passthrough:
             self.failure_handler(exception, traceback_exception)
@@ -2261,6 +2288,15 @@ class Logging(LiteLLMLoggingBaseClass):
         except Exception:
             return True
 
+    def claim_async_success_log(self) -> bool:
+        """One async success log per request. The innermost @client wrapper always exits first,
+        since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
+        False and schedules nothing."""
+        if self._async_success_scheduled:
+            return False
+        self._async_success_scheduled = True
+        return True
+
     def mark_logging_complete(
         self,
         event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
@@ -2304,10 +2340,12 @@ class Logging(LiteLLMLoggingBaseClass):
 
         return True
 
-    def _update_completion_start_time(self, completion_start_time: datetime.datetime):
+    def update_completion_start_time(self, completion_start_time: datetime.datetime) -> None:
         self.completion_start_time = completion_start_time
         self.model_call_details["completion_start_time"] = self.completion_start_time
 
+    _update_completion_start_time = update_completion_start_time
+
     def normalize_logging_result(self, result: object) -> object:
         """
         Some endpoints return a different type of result than what is expected by the logging system.
@@ -2379,7 +2417,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """
         if logging_result is None:
             return
-        hidden_params: Final = getattr(logging_result, "_hidden_params", None)
+        hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, None)
         if not hidden_params:
             return
         if self.model_call_details.get("litellm_params") is None:
@@ -2403,7 +2441,7 @@ class Logging(LiteLLMLoggingBaseClass):
     ):
         """Resolve hidden params, compute response cost, and emit the standard logging payload."""
         self._surface_response_headers_from_result(logging_result)
-        hidden_params: Final = getattr(logging_result, "_hidden_params", {})
+        hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {})
         if hidden_params:
             if self.model_call_details.get("litellm_params") is not None:
                 self.model_call_details["litellm_params"].setdefault("metadata", {})
@@ -2424,7 +2462,7 @@ class Logging(LiteLLMLoggingBaseClass):
             # Do not preserve 0 from failure_handler on intermediate router retries.
             pass
         else:
-            self.model_call_details["response_cost"] = self._response_cost_calculator(result=logging_result)
+            self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
 
         if not build_logging_payload:
             return
@@ -2472,7 +2510,7 @@ class Logging(LiteLLMLoggingBaseClass):
     def _transform_usage_objects(self, result):
         if isinstance(result, ResponsesAPIResponse):
             result = result.model_copy()
-            transformed_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(result.usage)
+            transformed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(result.usage)
             setattr(result, "usage", transformed_usage)
             if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
                 response_dict: Final = result.model_dump() if hasattr(result, "model_dump") else dict(result)
@@ -2678,7 +2716,7 @@ class Logging(LiteLLMLoggingBaseClass):
         Left in place they overwrite the create's real deployment with the
         poll's empty one in the payload every logging integration reads.
         """
-        settled_hidden_params: Final = getattr(result, "_hidden_params", None)
+        settled_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None)
         if isinstance(settled_hidden_params, dict):
             for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
                 settled_hidden_params.pop(poll_scoped_key, None)
@@ -2798,7 +2836,7 @@ class Logging(LiteLLMLoggingBaseClass):
             standard_logging_object=kwargs.get("standard_logging_object", None),
         )
         litellm_params = self.model_call_details.get("litellm_params", {})
-        is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
+        is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
         try:
             ## BUILD COMPLETE STREAMED RESPONSE
             complete_streaming_response: (
@@ -2817,7 +2855,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 verbose_logger.debug("Logging Details LiteLLM-Success Call streaming complete")
                 self.model_call_details["complete_streaming_response"] = complete_streaming_response
                 self._surface_response_headers_from_result(complete_streaming_response)
-                self.model_call_details["response_cost"] = self._response_cost_calculator(
+                self.model_call_details["response_cost"] = self.response_cost_calculator(
                     result=complete_streaming_response
                 )
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
@@ -3252,13 +3290,12 @@ class Logging(LiteLLMLoggingBaseClass):
             batch_successful_requests: Final = kwargs.get("batch_successful_requests", None)
             batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
             has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
-
             should_compute_batch_data: Final = not has_explicit_batch_data and batch_cost_is_final(result)
             if has_explicit_batch_data:
-                result._hidden_params["response_cost"] = batch_cost
-                result._hidden_params["batch_models"] = batch_models
-                result._hidden_params["batch_successful_requests"] = batch_successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above
-                result._hidden_params["batch_failed_requests"] = batch_failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
+                set_hidden_param(result, "response_cost", batch_cost)
+                set_hidden_param(result, "batch_models", batch_models)
+                set_hidden_param(result, "batch_successful_requests", batch_successful_requests)
+                set_hidden_param(result, "batch_failed_requests", batch_failed_requests)
                 result.usage = batch_usage
                 batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None)
                 batch_completion_cost: Final = kwargs.get("batch_completion_cost", None)
@@ -3275,7 +3312,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     )
 
             elif should_compute_batch_data:
-                batch_result: Final = await _handle_completed_batch(
+                batch_result: Final = await handle_completed_batch(
                     batch=result,
                     custom_llm_provider=self.custom_llm_provider,
                     model_name=self.get_deployment_model_for_cost(),
@@ -3283,10 +3320,10 @@ class Logging(LiteLLMLoggingBaseClass):
                     model_info=self.get_router_deployment_model_info(),
                 )
 
-                result._hidden_params["response_cost"] = batch_result.cost
-                result._hidden_params["batch_models"] = batch_result.models
-                result._hidden_params["batch_successful_requests"] = batch_result.successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
-                result._hidden_params["batch_failed_requests"] = batch_result.failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
+                set_hidden_param(result, "response_cost", batch_result.cost)
+                set_hidden_param(result, "batch_models", batch_result.models)
+                set_hidden_param(result, "batch_successful_requests", batch_result.successful_requests)
+                set_hidden_param(result, "batch_failed_requests", batch_result.failed_requests)
                 result.usage = batch_result.usage
                 self.set_cost_breakdown(
                     input_cost=batch_result.prompt_cost,
@@ -3341,9 +3378,9 @@ class Logging(LiteLLMLoggingBaseClass):
                     self.model_call_details["response_cost"] = 0.0
                 else:
                     # check if base_model set on azure
-                    _get_base_model_from_metadata(model_call_details=self.model_call_details)
+                    get_base_model_from_metadata(model_call_details=self.model_call_details)
                     # base_model defaults to None if not set on model_info
-                    self.model_call_details["response_cost"] = self._response_cost_calculator(
+                    self.model_call_details["response_cost"] = self.response_cost_calculator(
                         result=complete_streaming_response
                     )
 
@@ -3566,7 +3603,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     if self.stream:
                         if "async_complete_streaming_response" in self.model_call_details:
                             print_verbose("DynamoDB Logger: Got Stream Event - Completed Stream Response")
-                            await dynamoLogger._async_log_event(
+                            await dynamoLogger.async_log_event(
                                 kwargs=self.model_call_details,
                                 response_obj=self.model_call_details["async_complete_streaming_response"],
                                 start_time=start_time,
@@ -3576,7 +3613,7 @@ class Logging(LiteLLMLoggingBaseClass):
                         else:
                             print_verbose("DynamoDB Logger: Got Stream Event - No complete stream response as yet")
                     else:
-                        await dynamoLogger._async_log_event(
+                        await dynamoLogger.async_log_event(
                             kwargs=self.model_call_details,
                             response_obj=result,
                             start_time=start_time,
@@ -3603,7 +3640,7 @@ class Logging(LiteLLMLoggingBaseClass):
         try:
             callback_name: Final = self._get_callback_name(callback)
 
-            all_callbacks: Final = litellm.logging_callback_manager._get_all_callbacks()
+            all_callbacks: Final = litellm.logging_callback_manager.get_all_callbacks()
 
             for callback_obj in all_callbacks:
                 if hasattr(callback_obj, "increment_callback_logging_failure"):
@@ -3632,7 +3669,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["log_event_type"] = "failed_api_call"
         self.model_call_details["exception"] = exception
         self.model_call_details["traceback_exception"] = (
-            _redact_string(traceback_exception) if isinstance(traceback_exception, str) else traceback_exception
+            redact_string(traceback_exception) if isinstance(traceback_exception, str) else traceback_exception
         )
         self.model_call_details["end_time"] = end_time
         self.model_call_details.setdefault("original_response", None)
@@ -3657,7 +3694,7 @@ class Logging(LiteLLMLoggingBaseClass):
             end_time=end_time,
             logging_obj=self,
             status="failure",
-            error_str=_redact_string(str(exception)),
+            error_str=redact_string(str(exception)),
             original_exception=exception,
             standard_built_in_tools_params=self.standard_built_in_tools_params,
         )
@@ -3725,7 +3762,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if not self.should_run_logging(event_type="sync_failure"):  # prevent double logging
             return
         litellm_params: Final = self.model_call_details.get("litellm_params", {})
-        is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
+        is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
 
         try:
             start_time, end_time = self._failure_handler_helper_fn(
@@ -3977,7 +4014,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 self._handle_callback_failure(callback=callback)
         await flush_post_call_redis_batches()
 
-    def _get_trace_id(self, service_name: Literal["langfuse"]) -> str | None:
+    def get_trace_id(self, service_name: Literal["langfuse"]) -> str | None:
         """
         For the given service (e.g. langfuse), return the trace_id actually logged.
 
@@ -3995,6 +4032,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         return trace_id
 
+    _get_trace_id = get_trace_id
+
     def handle_sync_success_callbacks_for_async_calls(
         self,
         result: object,
@@ -4139,7 +4178,7 @@ class Logging(LiteLLMLoggingBaseClass):
             ## return unified Usage object
             if isinstance(result.response.usage, ResponseAPIUsage):
                 set_response_cost_in_hidden_params(result.response, result.response.usage.cost)
-                transformed_usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
+                transformed_usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(
                     result.response.usage
                 )
                 # Set as dict instead of Usage object so model_dump() serializes it correctly
@@ -4304,11 +4343,11 @@ class Logging(LiteLLMLoggingBaseClass):
             model_response: Final = litellm.ModelResponse(id=served_id)
             model_response.model = self.model
             usage: Final = getattr(result, "usage", None)
-            if usage is not None and ResponseAPILoggingUtils._is_response_api_usage(usage):
+            if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage):
                 setattr(
                     model_response,
                     "usage",
-                    ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage),
+                    ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage),
                 )
             return model_response
 
@@ -4358,15 +4397,15 @@ class Logging(LiteLLMLoggingBaseClass):
         return result_copy
 
 
-def _get_masked_values(
-    sensitive_object: dict,
+def get_masked_values(
+    sensitive_object: Mapping[str, object],
     ignore_sensitive_values: bool = False,
     mask_all_values: bool = False,
     unmasked_length: int = 4,
     number_of_asterisks: int | None = 4,
     _depth: int = 0,
     _max_depth: int = 20,
-) -> dict:
+) -> dict[str, object]:
     """
     Internal debugging helper function
 
@@ -4376,7 +4415,7 @@ def _get_masked_values(
         masked_length: Optional length for the masked portion (number of *). If set, will use exactly this many *
                      regardless of original string length. The total length will be unmasked_length + masked_length.
     """
-    sensitive_keywords: Final = [
+    sensitive_keywords: Final = (
         "authorization",
         "token",
         "key",
@@ -4385,14 +4424,17 @@ def _get_masked_values(
         "credentials",
         "password",
         "passwd",
-    ]
+    )
 
     def _mask_value(v: object) -> object:
         if isinstance(v, dict):
+            typed_value: Final = cast(  # cast-ok: request values can contain arbitrary header keys
+                dict[object, object], v
+            )
             if _depth >= _max_depth:
                 return v
-            return _get_masked_values(
-                v,
+            return get_masked_values(
+                cast(dict[str, object], typed_value),  # cast-ok: preserve dynamic key behavior
                 ignore_sensitive_values=ignore_sensitive_values,
                 mask_all_values=mask_all_values,
                 unmasked_length=unmasked_length,
@@ -4419,7 +4461,13 @@ def _get_masked_values(
     }
 
 
-def set_callbacks(callback_list, function_id=None):
+_get_masked_values = get_masked_values
+
+
+def set_callbacks(
+    callback_list: Iterable[str | Callable[..., object] | CustomLogger],
+    function_id: str | None = None,
+) -> None:
     """
     Globally sets the callback client
     """
@@ -4517,14 +4565,14 @@ def set_callbacks(callback_list, function_id=None):
 def _init_custom_logger_compatible_class(
     logging_integration: _custom_logger_compatible_callbacks_literal,
     internal_usage_cache: DualCache | None,
-    llm_router: object,  # expect litellm.Router, but typing errors due to circular import
-    custom_logger_init_args: dict | None = {},
+    llm_router: object,
+    custom_logger_init_args: dict[str, object] | None = {},
 ) -> CustomLogger | None:
     """
     Initialize a custom logger compatible class
     """
     try:
-        custom_logger_init_args = custom_logger_init_args or {}
+        custom_logger_init_args_value: Final[dict[str, object]] = custom_logger_init_args or {}
         if logging_integration == "agentops":  # Add AgentOps initialization
             _v2 = _maybe_construct_otel_v2("agentops", _in_memory_loggers)
             if _v2 is not None:
@@ -4614,10 +4662,10 @@ def _init_custom_logger_compatible_class(
             return _prometheus_logger
         elif logging_integration == "datadog":
             # Check if team-scoped credentials are provided
-            _dd_api_key: Final = custom_logger_init_args.get("dd_api_key")
-            _dd_site: Final = custom_logger_init_args.get("dd_site")
-            _dd_agent_host: Final = custom_logger_init_args.get("dd_agent_host")
-            _dd_agent_port: Final = custom_logger_init_args.get("dd_agent_port")
+            _dd_api_key: Final = custom_logger_init_args_value.get("dd_api_key")
+            _dd_site: Final = custom_logger_init_args_value.get("dd_site")
+            _dd_agent_host: Final = custom_logger_init_args_value.get("dd_agent_host")
+            _dd_agent_port: Final = custom_logger_init_args_value.get("dd_agent_port")
 
             if _dd_api_key or _dd_site or _dd_agent_host:
                 # Team-scoped credentials: use DynamicLoggingCache for per-credential isolation
@@ -4626,7 +4674,9 @@ def _init_custom_logger_compatible_class(
                 )
 
                 return DataDogHandler.get_datadog_logger_for_request(
-                    standard_callback_dynamic_params=custom_logger_init_args,
+                    standard_callback_dynamic_params=cast(  # cast-ok: callback options come from dynamic config
+                        StandardCallbackDynamicParams, custom_logger_init_args_value
+                    ),
                     in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
                 )
 
@@ -5068,7 +5118,7 @@ def _init_custom_logger_compatible_class(
             for callback in _in_memory_loggers:
                 if isinstance(callback, pagerduty_loggers.pagerduty):
                     return callback
-            pagerduty_logger: Final = pagerduty_loggers.pagerduty(**custom_logger_init_args)
+            pagerduty_logger: Final = pagerduty_loggers.pagerduty(**custom_logger_init_args_value)
             _in_memory_loggers.append(pagerduty_logger)
             return pagerduty_logger
         elif logging_integration == "anthropic_cache_control_hook":
@@ -5178,7 +5228,7 @@ def _init_custom_logger_compatible_class(
             _in_memory_loggers.append(gitlab_logger)
             return gitlab_logger
         elif logging_integration == "newrelic":
-            if custom_logger_init_args.get("newrelic_api_key"):
+            if custom_logger_init_args_value.get("newrelic_api_key"):
                 # Team-scoped credentials: per-team METRICS logger, isolated per
                 # credential set via DynamicLoggingCache. The trace logger for
                 # this name stays on the global path below.
@@ -5187,7 +5237,9 @@ def _init_custom_logger_compatible_class(
                 )
 
                 return NewRelicHandler.get_newrelic_logger_for_request(
-                    standard_callback_dynamic_params=custom_logger_init_args,
+                    standard_callback_dynamic_params=cast(  # cast-ok: callback options come from dynamic config
+                        StandardCallbackDynamicParams, custom_logger_init_args_value
+                    ),
                     in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
                 )
 
@@ -5350,7 +5402,7 @@ def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger])
 
 
 def get_custom_logger_compatible_class(
-    logging_integration: _custom_logger_compatible_callbacks_literal,
+    logging_integration: str,
 ) -> CustomLogger | None:
     try:
         if logging_integration == "lago":
@@ -5922,10 +5974,10 @@ class StandardLoggingPayloadSetup:
         elif isinstance(usage, Usage):
             return usage
         elif isinstance(usage, ResponseAPIUsage):
-            return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
+            return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
         elif isinstance(usage, dict):
-            if ResponseAPILoggingUtils._is_response_api_usage(usage):
-                return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
+            if ResponseAPILoggingUtils.is_response_api_usage(usage):
+                return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
             if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
                 return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
             return Usage(**usage)
@@ -5950,10 +6002,10 @@ class StandardLoggingPayloadSetup:
         if _raw is None:
             return _empty
         if isinstance(_raw, ResponseAPIUsage):
-            return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
+            return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
         if isinstance(_raw, dict):
-            if ResponseAPILoggingUtils._is_response_api_usage(_raw):
-                return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
+            if ResponseAPILoggingUtils.is_response_api_usage(_raw):
+                return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
             if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
                 return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
             return _raw
@@ -5969,7 +6021,7 @@ class StandardLoggingPayloadSetup:
         init_response_obj: object,
         api_base: str | None = None,
     ) -> StandardLoggingModelInformation:
-        model_cost_name: Final = _select_model_name_for_cost_calc(
+        model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
             completion_response=init_response_obj,
             base_model=base_model,
@@ -6196,8 +6248,8 @@ class StandardLoggingPayloadSetup:
             error_code=error_status,
             error_class=error_class,
             llm_provider=_llm_provider_in_exception,
-            traceback=_redact_string(traceback_info),
-            error_message=_redact_string(error_message),
+            traceback=redact_string(traceback_info),
+            error_message=redact_string(error_message),
             error_provider_request_id=provider_request_id,
             error_rate_limit_category=rate_limit_category,
             error_rate_limit_type=rate_limit_type,
@@ -6321,7 +6373,7 @@ class StandardLoggingPayloadSetup:
         return logging_obj.litellm_session_id
 
     @staticmethod
-    def _get_user_agent_tags(proxy_server_request: dict) -> list[str] | None:
+    def _get_user_agent_tags(proxy_server_request: Mapping[str, object]) -> list[str] | None:
         """
         Return the user agent tags from the proxy server request for spend tracking
         """
@@ -6330,8 +6382,11 @@ class StandardLoggingPayloadSetup:
         user_agent_tags: list[str] | None = None
         headers: Final = proxy_server_request.get("headers", {})
         if headers is not None and isinstance(headers, dict):
-            if "user-agent" in headers:
-                user_agent: Final = headers["user-agent"]
+            request_headers: Final = cast(  # cast-ok: request headers are untyped framework data
+                dict[str, str | None], headers
+            )
+            if "user-agent" in request_headers:
+                user_agent: Final = request_headers["user-agent"]
                 if user_agent is not None:
                     if user_agent_tags is None:
                         user_agent_tags = []
@@ -6340,12 +6395,11 @@ class StandardLoggingPayloadSetup:
                         user_agent_part = user_agent.split("/")[0]
                     if user_agent_part is not None:
                         user_agent_tags.append("User-Agent: " + user_agent_part)
-                    if user_agent is not None:
-                        user_agent_tags.append("User-Agent: " + user_agent)
+                    user_agent_tags.append("User-Agent: " + user_agent)
         return user_agent_tags
 
     @staticmethod
-    def _get_extra_header_tags(proxy_server_request: dict) -> list[str] | None:
+    def _get_extra_header_tags(proxy_server_request: Mapping[str, object]) -> list[str] | None:
         """
         Extract additional header tags for spend tracking based on config.
         """
@@ -6356,24 +6410,33 @@ class StandardLoggingPayloadSetup:
         headers: Final = proxy_server_request.get("headers", {})
         if not isinstance(headers, dict):
             return None
+        request_headers: Final = cast(  # cast-ok: request headers are untyped framework data
+            dict[str, str], headers
+        )
 
         header_tags: Final = []
         for header_name in extra_headers:
-            header_value = headers.get(header_name)
+            header_value = request_headers.get(header_name)
             if header_value:
                 header_tags.append(f"{header_name}: {header_value}")
 
         return header_tags if header_tags else None
 
     @staticmethod
-    def _get_request_tags(litellm_params: dict, proxy_server_request: dict) -> list[str]:
-        # check for 'tags' in both 'metadata' and 'litellm_metadata'
-        metadata: Final = litellm_params.get("metadata") or {}
-        litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
+    def get_request_tags(
+        litellm_params: dict[str, object],
+        proxy_server_request: dict[str, object],
+    ) -> list[str]:
+        metadata: Final = cast(  # cast-ok: request metadata is caller-provided
+            Mapping[str, object], litellm_params.get("metadata") or {}
+        )
+        litellm_metadata: Final = cast(  # cast-ok: request metadata is caller-provided
+            Mapping[str, object], litellm_params.get("litellm_metadata") or {}
+        )
         if metadata.get("tags", []):
-            request_tags = metadata.get("tags", []).copy()
+            request_tags = cast(list[str], metadata.get("tags", [])).copy()  # cast-ok: tags are caller-provided
         elif litellm_metadata.get("tags", []):
-            request_tags = litellm_metadata.get("tags", []).copy()
+            request_tags = cast(list[str], litellm_metadata.get("tags", [])).copy()  # cast-ok: tags are caller-provided
         else:
             request_tags = []
         user_agent_tags: Final = StandardLoggingPayloadSetup._get_user_agent_tags(proxy_server_request)
@@ -6384,6 +6447,8 @@ class StandardLoggingPayloadSetup:
             request_tags.extend(additional_header_tags)
         return request_tags
 
+    _get_request_tags = get_request_tags
+
 
 def _get_status_fields(
     status: StandardLoggingPayloadStatus,
@@ -6451,7 +6516,7 @@ def _extract_response_obj_and_hidden_params(
 ) -> tuple[dict, dict | None]:
     """Extract response_obj and hidden_params from init_response_obj."""
     hidden_params: dict | None = (
-        getattr(init_response_obj, "_hidden_params", None)
+        getattr(init_response_obj, HIDDEN_PARAMS_ATTR, None)
         if isinstance(init_response_obj, BaseModel | HttpxBinaryResponseContent)
         else None
     )
@@ -6565,7 +6630,7 @@ def get_standard_logging_object_payload(
         _model_id: Final = metadata.get("model_info", {}).get("id", "")
         _model_group: Final = metadata.get("model_group", "")
 
-        request_tags: Final = StandardLoggingPayloadSetup._get_request_tags(
+        request_tags: Final = StandardLoggingPayloadSetup.get_request_tags(
             litellm_params=litellm_params, proxy_server_request=proxy_server_request
         )
         request_model_access_groups: Final = request_model_access_groups_from_litellm_params(litellm_params)
@@ -6610,7 +6675,7 @@ def get_standard_logging_object_payload(
         if cache_hit is True:
             id = f"{id}_cache_hit{time.time()}"  # do not duplicate the request id
             saved_cache_cost = (
-                logging_obj._response_cost_calculator(
+                logging_obj.response_cost_calculator(
                     result=init_response_obj,
                     cache_hit=False,
                 )
@@ -6618,7 +6683,7 @@ def get_standard_logging_object_payload(
             )
 
         ## Get model cost information ##
-        base_model = _get_base_model_from_metadata(model_call_details=kwargs)
+        base_model = get_base_model_from_metadata(model_call_details=kwargs)
         # The router overrides completion_response.model to the model-group alias before
         # this payload is built, so cost-map lookup via that alias always misses.
         # Fall back to the actual deployment model set by the router in metadata.
@@ -6671,7 +6736,9 @@ def get_standard_logging_object_payload(
         # Reconstruct full model name with provider prefix for logging
         # This ensures Bedrock models like "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
         # are logged as "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
-        custom_llm_provider: Final = cast(str | None, kwargs.get("custom_llm_provider"))
+        custom_llm_provider: Final = cast(  # cast-ok: provider name is caller-provided
+            str | None, kwargs.get("custom_llm_provider")
+        )
         model_name = reconstruct_model_name(kwargs.get("model", "") or "", custom_llm_provider, metadata)
         response_model_name: str | None = None
         if isinstance(final_response_obj, dict):
@@ -6923,7 +6990,7 @@ def _get_traceback_str_for_error(error_str: str) -> str:
 from decimal import Decimal
 
 # used for unit testing
-from typing import Any, Union
+from typing import Any
 
 
 def create_dummy_standard_logging_payload() -> StandardLoggingPayload: