scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge branch 'litellm_internal_staging' into litellm_fix_bedrock_adaptive_thinking_token_accounting

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-08-05 21:13:36 +00:00
commit aadfa89ff6
934 changed files with 25797 additions and 13583 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
.flake8
View file

@ -1,46 +0,0 @@
[flake8]
ignore =
# The following ignores can be removed when formatting using black
W191,W291,W292,W293,W391,W504
E101,E111,E114,E116,E117,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,
E201,E202,E221,E222,E225,E226,E231,E241,E251,E252,E261,E265,E271,E272,E275,
E301,E302,E303,E305,E306,
# line break before binary operator
W503,
# inline comment should start with '# '
E262,
# too many leading '#' for block comment
E266,
# multiple imports on one line
E401,
# module level import not at top of file
E402,
# Line too long (82 > 79 characters)
E501,
# comparison to None should be 'if cond is None:'
E711,
# comparison to True should be 'if cond is True:' or 'if cond:'
E712,
# do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
E721,
# do not use bare 'except'
E722,
# x is imported but unused
F401,
# 'from . import *' used; unable to detect undefined names
F403,
# x may be undefined, or defined from star imports:
F405,
# f-string is missing placeholders
F541,
# dictionary key '' repeated with different values
F601,
# redefinition of unused x from line 123
F811,
# undefined name x
F821,
# local variable x is assigned to but never used
F841,
# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
extend-ignore = E203

13
.github/workflows/_test-unit-base.yml vendored
View file

@ -154,6 +154,19 @@ jobs:
merge-multiple: true
- name: Upload to Codecov
id: codecov-upload
continue-on-error: true
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false
- name: Upload to Codecov (retry)
if: steps.codecov-upload.outcome == 'failure'
continue-on-error: true
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true

3
.github/workflows/image-scan.yml vendored
View file

@ -11,7 +11,8 @@ on:
- docker/Dockerfile.non_root
- migrations/Dockerfile
- migrations/run.py
- tests/proxy_migration_tests/test_offline_image_migration.py
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- .github/workflows/image-scan.yml

42
.github/workflows/test-linting.yml vendored
View file

@ -23,10 +23,21 @@ jobs:
# Any-discipline) would otherwise blame on this branch.
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
fetch-depth: 1
clean: true
persist-credentials: false
- name: Fetch gate base (merge-base with target branch)
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
MERGE_BASE=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$MERGE_BASE"
git fetch --no-tags --depth=1 origin "$MERGE_BASE"
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
@ -60,10 +71,8 @@ jobs:
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Check ruff format
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
echo "No changed litellm Python files to check with ruff format."
exit 0
@ -86,33 +95,24 @@ jobs:
cd ..
- name: Check strict-rule budget (delta vs base)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Print OpenAI version
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Check basedpyright budget (delta vs base)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
NODE_OPTIONS: --max-old-space-size=12288
run: |
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
- name: Check tests/e2e basedpyright (zero errors)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
uv run --no-sync basedpyright tests/e2e
else
echo "No changed tests/e2e Python files; skipping."
@ -141,9 +141,15 @@ jobs:
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0
fetch-depth: 1
persist-credentials: false
- name: Fetch ratchet base
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
git fetch --no-tags --depth=1 origin "$BASE_SHA"
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:

4
CLAUDE.md
View file

@ -39,10 +39,6 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Python max line length is 120, not 88
On a fresh worktree or clone, run `make bootstrap` before anything else. It provisions everything tests, `make pre-commit`, and a local proxy need
Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit
When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in

3
Dockerfile
View file

@ -134,7 +134,8 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
EXPOSE 4000/tcp

8
Makefile
View file

@ -75,7 +75,7 @@ install-dev:
bootstrap:
$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
$(UV_RUN) python scripts/prisma_generate_if_needed.py
cd ui/litellm-dashboard && npm install --no-audit --no-fund
cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund
@main_root=$$(git worktree list --porcelain | head -1 | sed 's/^worktree //'); \
if [ "$$main_root" != "$$(git rev-parse --show-toplevel)" ] && [ -f "$$main_root/.env" ] && [ ! -f .env ]; then \
cp "$$main_root/.env" .env && echo "bootstrap: copied .env from $$main_root"; \
@ -176,10 +176,8 @@ lint-ruff-FULL-dev: install-dev
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
else echo "No changed .py files to check."; fi
lint-basedpyright lint-basedpyright-budget-update: export NODE_OPTIONS := --max-old-space-size=12288
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
$(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e
@ -192,7 +190,7 @@ lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev lint-fetch-base
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
$(UV_RUN) python scripts/type_check_gate.py --update
lint-format: format-check

15
backend/Dockerfile
View file

@ -59,9 +59,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
@ -83,13 +83,16 @@ ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
USER nonroot

4
backend/routes/allowlist.py
View file

@ -44,6 +44,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/router/",
"/router_settings",
"/adaptive_router/",
"/auto_router/",
"/fallback",
"/fallbacks",
"/cache_settings",
@ -81,6 +82,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/user_agent",
"/usage/",
"/daily/",
# Deployment-wide gateway request counts. Scoped to the analytics read rather
# than all of /gateway/, which stays free for data-plane routes.
"/gateway/daily/",
# CloudZero cost-export admin (init / settings / export / dry-run / delete)
"/cloudzero/",
# Caching admin

48
basedpyright-code-budget.json
View file

@ -1,9 +1,9 @@
{
"reportAny": {
"limit": 29809
"limit": 29204
},
"reportArgumentType": {
"limit": 2645
"limit": 2635
},
"reportAssignmentType": {
"limit": 329
@ -18,28 +18,28 @@
"limit": 40
},
"reportDeprecated": {
"limit": 325
"limit": 215
},
"reportDuplicateImport": {
"limit": 24
"limit": 19
},
"reportExplicitAny": {
"limit": 9473
"limit": 9227
},
"reportFunctionMemberAccess": {
"limit": 11
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 157
},
"reportIncompatibleMethodOverride": {
"limit": 77
"limit": 56
},
"reportIncompatibleVariableOverride": {
"limit": 12
"limit": 8
},
"reportInconsistentOverload": {
"limit": 18
"limit": 12
},
"reportIndexIssue": {
"limit": 35
@ -48,16 +48,16 @@
"limit": 35
},
"reportInvalidTypeVarUse": {
"limit": 5
"limit": 2
},
"reportMatchNotExhaustive": {
"limit": 0
},
"reportMissingParameterType": {
"limit": 5855
"limit": 5850
},
"reportMissingTypeArgument": {
"limit": 15849
"limit": 15833
},
"reportMissingTypeStubs": {
"limit": 40
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1079
"limit": 1078
},
"reportOptionalOperand": {
"limit": 0
@ -84,13 +84,13 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 2436
"limit": 1825
},
"reportRedeclaration": {
"limit": 8
},
"reportReturnType": {
"limit": 219
"limit": 218
},
"reportTypedDictNotRequiredAccess": {
"limit": 27
@ -99,22 +99,22 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 45262
"limit": 45242
},
"reportUnknownLambdaType": {
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40452
"limit": 40340
},
"reportUnknownParameterType": {
"limit": 20309
"limit": 20293
},
"reportUnknownVariableType": {
"limit": 31978
"limit": 31796
},
"reportUnnecessaryCast": {
"limit": 124
"limit": 122
},
"reportUnnecessaryComparison": {
"limit": 703
@ -123,10 +123,10 @@
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 866
"limit": 865
},
"reportUntypedBaseClass": {
"limit": 165
"limit": 72
},
"reportUntypedFunctionDecorator": {
"limit": 33
@ -138,9 +138,9 @@
"limit": 139
},
"reportUnusedImport": {
"limit": 588
"limit": 555
},
"reportUnusedVariable": {
"limit": 147
"limit": 146
}
}

3
docker/Dockerfile.database
View file

@ -133,7 +133,8 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
EXPOSE 4000/tcp

3
docker/Dockerfile.non_root
View file

@ -185,7 +185,8 @@ RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/u
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1
ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
USER 65534

8
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -47,7 +47,13 @@ RUN uv venv --python python && \
"prisma==0.11.0" \
"openai==2.24.0"
RUN prisma generate --schema=./schema.prisma
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma && \
chmod -R a+rX /opt/prisma && \
python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None"
ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
EXPOSE 4000/tcp

0
enterprise/litellm_enterprise/enterprise_callbacks/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/integrations/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/litellm_core_utils/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/proxy/hooks/__init__.py Normal file
View file

6
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -382,7 +382,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids]
return [
OpenAIFileObject.model_validate(file_object.file_object)
for file_object in file_ids
if file_object.file_object is not None
]
async def check_managed_file_id_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth

4
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -831,7 +831,7 @@ async def project_info(
)
# Check if user has access to this project (admin or team member)
is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
is_admin = user_api_key_has_admin_view(user_api_key_dict)
is_team_member = False
if project.team_id and user_api_key_dict.user_id:
@ -886,7 +886,7 @@ async def list_projects(
)
# If proxy admin, get all projects
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
if user_api_key_has_admin_view(user_api_key_dict):
projects: Sequence[
prisma_models.LiteLLM_ProjectTable
] = await prisma_client.db.litellm_projecttable.find_many(

0
enterprise/litellm_enterprise/proxy/vector_stores/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/py.typed Normal file
View file

0
enterprise/litellm_enterprise/types/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/types/enterprise_callbacks/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/types/proxy/__init__.py Normal file
View file

15
gateway/Dockerfile
View file

@ -61,9 +61,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
@ -85,13 +85,16 @@ ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
USER nonroot

15
litellm-proxy-extras/litellm_proxy_extras/migrations/20260803000000_add_daily_gateway_requests/migration.sql Normal file
View file

@ -0,0 +1,15 @@
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGatewayRequests" (
"date" TEXT NOT NULL,
"category" TEXT NOT NULL,
"route" TEXT NOT NULL,
"successful_requests" BIGINT NOT NULL DEFAULT 0,
"failed_requests" BIGINT NOT NULL DEFAULT 0,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LiteLLM_DailyGatewayRequests_pkey" PRIMARY KEY ("date","category","route")
);
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_DailyGatewayRequests_date_idx" ON "LiteLLM_DailyGatewayRequests"("date");

31
litellm-proxy-extras/litellm_proxy_extras/migrations/20260805000000_add_autorouter_session_rollup/migration.sql Normal file
View file

@ -0,0 +1,31 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterSession" (
"api_key" TEXT NOT NULL,
"session_id" TEXT NOT NULL,
"router_name" TEXT NOT NULL,
"router_type" TEXT NOT NULL,
"first_turn_at" TIMESTAMP(3) NOT NULL,
"last_turn_at" TIMESTAMP(3) NOT NULL,
"last_model" TEXT NOT NULL,
"models" JSONB NOT NULL DEFAULT '{}',
"turns" INTEGER NOT NULL DEFAULT 0,
"unordered_turns" INTEGER NOT NULL DEFAULT 0,
"covered_turns" INTEGER NOT NULL DEFAULT 0,
"cache_hits" INTEGER NOT NULL DEFAULT 0,
"same_model_turns" INTEGER NOT NULL DEFAULT 0,
"same_model_hits" INTEGER NOT NULL DEFAULT 0,
"first_visit_turns" INTEGER NOT NULL DEFAULT 0,
"first_visit_hits" INTEGER NOT NULL DEFAULT 0,
"return_turns" INTEGER NOT NULL DEFAULT 0,
"return_hits" INTEGER NOT NULL DEFAULT 0,
"return_expired_misses" INTEGER NOT NULL DEFAULT 0,
"return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
"ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
"ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
"total_tokens" BIGINT NOT NULL DEFAULT 0,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
"saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
CONSTRAINT "LiteLLM_AutoRouterSession_pkey" PRIMARY KEY ("api_key", "session_id", "router_name")
);
CREATE INDEX IF NOT EXISTS "idx_autorouter_session_last_turn" ON "LiteLLM_AutoRouterSession"("last_turn_at");

181
litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py Normal file
View file

@ -0,0 +1,181 @@
"""Prepare the Node toolchain the Prisma CLI needs, separately from migrations.
The Prisma CLI is a Node program. The first invocation inside a fresh
container installs a private Node runtime and npm-installs the CLI itself,
which can take minutes on a cold or slow machine. Sharing one timeout between
that one-time bootstrap and the migration commands makes a slow bootstrap
indistinguishable from a slow migration, so the bootstrap gets killed long
before it can finish.
A killed bootstrap does not correct itself. The installer leaves its cache
directory behind, and Prisma decides whether to install by testing that
directory for existence alone, so every later attempt skips the install and
then fails on a Node binary that was never written. Deleting a cache directory
that exists without a Node binary is what turns a killed bootstrap back into a
recoverable one.
Both budgets are overridable so an operator can widen them without a release:
``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install and
``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every individual Prisma command.
"""
import math
import os
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
from litellm_proxy_extras._logging import logger
try:
from prisma import config as prisma_config
except ImportError:
prisma_config = None
PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
BOOTSTRAP_ARG = "--version"
@dataclass(frozen=True)
class ToolchainBootstrap:
"""Outcome of preparing the Prisma toolchain."""
healed_incomplete_cache: bool
ready: bool
def _timeout_from_env(env_var: str, default: float) -> float:
raw = os.getenv(env_var)
if raw is None:
return default
try:
seconds = float(raw)
except ValueError:
logger.warning(
"%s=%r is not a number, falling back to %ss", env_var, raw, default
)
return default
if not math.isfinite(seconds) or seconds <= 0:
logger.warning(
"%s=%r is not a finite positive number, falling back to %ss",
env_var,
raw,
default,
)
return default
return seconds
def prisma_command_timeout() -> float:
"""Seconds any single Prisma command may run for."""
return _timeout_from_env(
PRISMA_COMMAND_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_COMMAND_TIMEOUT
)
def prisma_bootstrap_timeout() -> float:
"""Seconds the one-time Node toolchain install may run for."""
return _timeout_from_env(
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT
)
def nodeenv_cache_dir() -> Optional[Path]:
"""Where Prisma installs its private Node runtime, or None if unknowable."""
override = os.getenv(NODEENV_CACHE_DIR_ENV_VAR)
if override:
return Path(override).absolute()
if prisma_config is not None:
try:
return Path(prisma_config.nodeenv_cache_dir).absolute()
except (OSError, ValueError) as e:
logger.warning("Could not read the Prisma nodeenv cache dir: %s", e)
try:
return Path.home() / ".cache" / "prisma-python" / "nodeenv"
except RuntimeError:
logger.warning(
"No resolvable home directory, cannot locate the Prisma nodeenv cache"
)
return None
def node_binary_path(cache_dir: Path) -> Path:
"""Path the Node binary occupies once the toolchain is fully installed."""
if os.name == "nt":
return cache_dir / "Scripts" / "node.exe"
return cache_dir / "bin" / "node"
def heal_incomplete_nodeenv_cache() -> bool:
"""Delete a nodeenv cache directory left without a Node binary.
Returns True when a half-installed toolchain was removed, so the next
Prisma invocation reinstalls it instead of failing on a missing binary.
"""
cache_dir = nodeenv_cache_dir()
if cache_dir is None:
return False
try:
if not cache_dir.is_dir() or node_binary_path(cache_dir).exists():
return False
except OSError as e:
logger.warning("Could not inspect the Node toolchain at %s: %s", cache_dir, e)
return False
logger.warning(
"Node toolchain at %s has no %s, so a previous install was interrupted. "
"Removing it so it can be reinstalled.",
cache_dir,
node_binary_path(cache_dir).name,
)
try:
shutil.rmtree(cache_dir)
except OSError as e:
logger.warning("Could not remove %s: %s", cache_dir, e)
return False
return True
def ensure_prisma_toolchain(
prisma_command: str, prisma_env: dict[str, str]
) -> ToolchainBootstrap:
"""Install whatever the Prisma CLI needs to run, under its own timeout.
Never raises. A toolchain that cannot be prepared is reported so the
caller can go on and let the real Prisma command produce the real error.
"""
healed = heal_incomplete_nodeenv_cache()
timeout = prisma_bootstrap_timeout()
logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout)
try:
subprocess.run(
[prisma_command, BOOTSTRAP_ARG],
timeout=timeout,
check=True,
capture_output=True,
text=True,
env=prisma_env,
)
except subprocess.TimeoutExpired:
logger.warning(
"Preparing the Prisma CLI toolchain timed out after %ss. Raise %s "
"if this machine needs longer to install it.",
timeout,
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR,
)
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=False)
except subprocess.CalledProcessError as e:
logger.warning("Preparing the Prisma CLI toolchain failed: %s", e.stderr)
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=False)
except OSError as e:
logger.warning("Could not run the Prisma CLI: %s", e)
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=False)
logger.info("Prisma CLI toolchain ready")
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=True)

0
litellm-proxy-extras/litellm_proxy_extras/py.typed Normal file
View file

3
litellm-proxy-extras/litellm_proxy_extras/replica_identity.py
View file

@ -16,6 +16,7 @@ import tempfile
from pathlib import Path
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout
REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL"
@ -75,7 +76,7 @@ def apply_replica_identity_full(
"--schema",
schema_path,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,

51
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1118,6 +1118,26 @@ model LiteLLM_DailyToolSpend {
@@id([date, tool_name])
}
// Gateway request counts recorded at the ASGI edge by
// BillableRequestMetricsMiddleware. This is the source of truth for SGR
// (successful gateway requests): it counts what the proxy actually answered,
// independent of whether the request reached litellm's logging callbacks.
// The key carries no deployment or caller dimension. Every part of it is
// chosen by the proxy and drawn from a closed set, so the table is bounded by
// (days x categories x routes) rather than by anything a caller can vary.
model LiteLLM_DailyGatewayRequests {
date String
category String
route String
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@id([date, category, route])
@@index([date])
}
// Prompt table for storing prompt configurations
model LiteLLM_PromptTable {
id String @id @default(uuid())
@ -1393,6 +1413,37 @@ model LiteLLM_AdaptiveRouterSession {
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
}
model LiteLLM_AutoRouterSession {
api_key String
session_id String
router_name String
router_type String
first_turn_at DateTime
last_turn_at DateTime
last_model String
models Json @default("{}")
turns Int @default(0)
unordered_turns Int @default(0)
covered_turns Int @default(0)
cache_hits Int @default(0)
same_model_turns Int @default(0)
same_model_hits Int @default(0)
first_visit_turns Int @default(0)
first_visit_hits Int @default(0)
return_turns Int @default(0)
return_hits Int @default(0)
return_expired_misses Int @default(0)
return_within_ttl_misses Int @default(0)
ttl_5m_turns Int @default(0)
ttl_1h_turns Int @default(0)
total_tokens BigInt @default(0)
spend Float @default(0)
saved_spend Float @default(0)
@@id([api_key, session_id, router_name])
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

33
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -14,6 +14,10 @@ from litellm_proxy_extras.replica_identity import (
REPLICA_IDENTITY_FULL_ENV_VAR,
apply_replica_identity_full,
)
from litellm_proxy_extras.prisma_toolchain import (
ensure_prisma_toolchain,
prisma_command_timeout,
)
def str_to_bool(value: Optional[str]) -> bool:
@ -142,7 +146,7 @@ class ProxyExtrasDBManager:
],
stdout=open(migration_file, "w"),
check=True,
timeout=30,
timeout=prisma_command_timeout(),
env=prisma_env,
)
@ -157,7 +161,7 @@ class ProxyExtrasDBManager:
"0_init",
],
check=True,
timeout=30,
timeout=prisma_command_timeout(),
env=prisma_env,
)
@ -193,7 +197,7 @@ class ProxyExtrasDBManager:
"--rolled-back",
migration_name,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
env=prisma_env,
@ -205,7 +209,7 @@ class ProxyExtrasDBManager:
prisma_env = _get_prisma_env()
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
env=prisma_env,
@ -303,7 +307,7 @@ class ProxyExtrasDBManager:
"--script",
],
check=True,
timeout=60,
timeout=prisma_command_timeout(),
stdout=f,
env=_get_prisma_env(),
)
@ -335,7 +339,7 @@ class ProxyExtrasDBManager:
"--schema",
schema_path,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -364,7 +368,7 @@ class ProxyExtrasDBManager:
"--schema",
schema_path,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -393,7 +397,7 @@ class ProxyExtrasDBManager:
"--applied",
migration_name,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -530,7 +534,7 @@ class ProxyExtrasDBManager:
try:
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
env=_get_prisma_env(),
)
@ -555,7 +559,7 @@ class ProxyExtrasDBManager:
try:
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -731,6 +735,9 @@ class ProxyExtrasDBManager:
Returns:
bool: True if setup was successful, False otherwise
"""
ensure_prisma_toolchain(
prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env()
)
migrated = ProxyExtrasDBManager._run_migrations(
use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
)
@ -757,7 +764,7 @@ class ProxyExtrasDBManager:
# Set migrations directory for Prisma
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -840,7 +847,7 @@ class ProxyExtrasDBManager:
"--rolled-back",
failed_migration,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -968,7 +975,7 @@ class ProxyExtrasDBManager:
# Use prisma db push with increased timeout
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
)
return True

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.82"
version = "0.4.83"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.82"
version = "0.4.83"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

44
litellm/__init__.py
View file

@ -1269,8 +1269,8 @@ from .llms.xai.common_utils import XAIModelInfo
from litellm.types.utils import LlmProviders
## Lazy loading this is not straightforward, will leave it here for now.
from .main import * # type: ignore
from .compression import compress # type: ignore[no-redef]
from .main import *
from .compression import compress
# Skills API
from .skills.main import (
@ -1341,7 +1341,7 @@ from .assistants.main import *
from .batches.main import *
from .images.main import *
from .videos.main import *
from .batch_completion.main import * # type: ignore
from .batch_completion.main import *
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
@ -2054,7 +2054,7 @@ if TYPE_CHECKING:
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType] # type: ignore[no-redef]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
validate_environment: Callable[..., dict]
check_valid_key: Callable[..., bool]
@ -2150,9 +2150,9 @@ def __getattr__(name: str) -> Any:
# Lazy load encoding from main.py to avoid heavy tiktoken import
if name == "encoding":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "encoding" not in _globals:
from .main import encoding as _encoding
@ -2162,9 +2162,9 @@ def __getattr__(name: str) -> Any:
# Lazy load bedrock_tool_name_mappings instance
if name == "bedrock_tool_name_mappings":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "bedrock_tool_name_mappings" not in _globals:
from .llms.bedrock.chat.invoke_handler import (
@ -2176,9 +2176,9 @@ def __getattr__(name: str) -> Any:
# Lazy load AzureOpenAIError exception class
if name == "AzureOpenAIError":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "AzureOpenAIError" not in _globals:
from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
@ -2188,9 +2188,9 @@ def __getattr__(name: str) -> Any:
# Lazy load openaiOSeriesConfig instance
if name == "openaiOSeriesConfig":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
if "openaiOSeriesConfig" not in _globals:
# Import the config class and instantiate it
config_class = __getattr__("OpenAIOSeriesConfig")
@ -2206,9 +2206,9 @@ def __getattr__(name: str) -> Any:
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
}
if name in _config_instances:
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
if name not in _globals:
# Import the config class and instantiate it
config_class = __getattr__(_config_instances[name])
@ -2221,9 +2221,9 @@ def __getattr__(name: str) -> Any:
# Lazy load provider_list
if name == "provider_list":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "provider_list" not in _globals:
# LlmProviders is eagerly imported above, so we can import it directly
@ -2234,9 +2234,9 @@ def __getattr__(name: str) -> Any:
# Lazy load priority_reservation_settings instance
if name == "priority_reservation_settings":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "priority_reservation_settings" not in _globals:
# Import the class and instantiate it
@ -2246,9 +2246,9 @@ def __getattr__(name: str) -> Any:
# Lazy load logging_callback_manager instance
if name == "logging_callback_manager":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "logging_callback_manager" not in _globals:
# Import the class and instantiate it
@ -2258,9 +2258,9 @@ def __getattr__(name: str) -> Any:
# Lazy load _service_logger module
if name == "_service_logger":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "_service_logger" not in _globals:
# Import the module lazily

10
litellm/_lazy_imports.py
View file

@ -54,7 +54,7 @@ from ._lazy_imports_registry import (
)
def _get_litellm_globals() -> dict:
def get_litellm_globals() -> dict:
"""
Get the globals dictionary of the litellm module.
@ -233,7 +233,7 @@ def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], cate
raise AttributeError(f"{category} lazy import: unknown attribute {name!r}")
# Step 2: Get the cache (where we store imported things)
_globals: Final = _get_litellm_globals()
_globals: Final = get_litellm_globals()
# Step 3: If we've already imported it, just return the cached version
if name in _globals:
@ -332,7 +332,7 @@ def _lazy_import_utils_module(name: str) -> Any:
Handler for utils module lazy imports.
This uses a custom implementation because utils module needs to use
_get_utils_globals() instead of _get_litellm_globals() for caching.
_get_utils_globals() instead of get_litellm_globals() for caching.
"""
# Check if this attribute exists in our map
if name not in _UTILS_MODULE_IMPORT_MAP:
@ -379,7 +379,7 @@ def _lazy_import_llm_client_cache(name: str) -> Any:
- "in_memory_llm_clients_cache" is a singleton instance of that class
So we need custom logic to handle both cases.
"""
_globals: Final = _get_litellm_globals()
_globals: Final = get_litellm_globals()
# If already cached, return it
if name in _globals:
@ -412,7 +412,7 @@ def _lazy_import_http_handlers(name: str) -> Any:
- They need configuration (timeout, etc.) from the module globals
- They use factory functions instead of direct instantiation
"""
_globals: Final = _get_litellm_globals()
_globals: Final = get_litellm_globals()
if name == "module_level_aclient":
# Create an async HTTP client using the factory function

30
litellm/_lazy_imports_registry.py
View file

@ -1461,32 +1461,30 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
# Export all name tuples and import maps for use in _lazy_imports.py
__all__ = [
# Name tuples
"COST_CALCULATOR_NAMES",
"LITELLM_LOGGING_NAMES",
"UTILS_NAMES",
"TOKEN_COUNTER_NAMES",
"LLM_CLIENT_CACHE_NAMES",
"BEDROCK_TYPES_NAMES",
"TYPES_UTILS_NAMES",
"CACHING_NAMES",
"HTTP_HANDLER_NAMES",
"COST_CALCULATOR_NAMES",
"DOTPROMPT_NAMES",
"HTTP_HANDLER_NAMES",
"LITELLM_LOGGING_NAMES",
"LLM_CLIENT_CACHE_NAMES",
"LLM_CONFIG_NAMES",
"TYPES_NAMES",
"LLM_PROVIDER_LOGIC_NAMES",
"TOKEN_COUNTER_NAMES",
"TYPES_NAMES",
"TYPES_UTILS_NAMES",
"UTILS_MODULE_NAMES",
# Import maps
"_UTILS_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
"_TYPES_UTILS_IMPORT_MAP",
"_TOKEN_COUNTER_IMPORT_MAP",
"UTILS_NAMES",
"_BEDROCK_TYPES_IMPORT_MAP",
"_CACHING_IMPORT_MAP",
"_LITELLM_LOGGING_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
"_DOTPROMPT_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_LITELLM_LOGGING_IMPORT_MAP",
"_LLM_CONFIGS_IMPORT_MAP",
"_LLM_PROVIDER_LOGIC_IMPORT_MAP",
"_TOKEN_COUNTER_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_TYPES_UTILS_IMPORT_MAP",
"_UTILS_IMPORT_MAP",
"_UTILS_MODULE_IMPORT_MAP",
]

28
litellm/_redis.py
View file

@ -15,8 +15,8 @@ import os
from collections.abc import Callable
from typing import Final
import redis # type: ignore
import redis.asyncio as async_redis # type: ignore
import redis
import redis.asyncio as async_redis
from litellm import get_secret, get_secret_str
from litellm._redis_credential_provider import (
@ -153,7 +153,7 @@ def _redis_kwargs_from_environment():
return_dict: Final = {}
for k, v in mapping.items():
value = get_secret(k, default_value=None) # type: ignore
value = get_secret(k, default_value=None)
if value is not None:
return_dict[v] = value
return return_dict
@ -317,7 +317,7 @@ def create_azure_ad_redis_connect_func(
# AzureADCredentialProvider for refresh-aware token retrieval. The raw
# client_id/tenant_id/secret are intentionally NOT exposed here — the
# credential closure already holds them.
ad_connect._azure_credential = credential # type: ignore[attr-defined]
ad_connect._azure_credential = credential
return ad_connect
@ -351,7 +351,7 @@ def _get_redis_client_logic(**env_overrides):
for k, v in env_overrides.items():
if isinstance(v, str) and v.startswith("os.environ/"):
v = v.replace("os.environ/", "")
value = get_secret(v) # type: ignore
value = get_secret(v)
env_overrides[k] = value
environment_kwargs: Final = _redis_kwargs_from_environment()
@ -370,7 +370,7 @@ def _get_redis_client_logic(**env_overrides):
**env_overrides,
}
_startup_nodes: Final[str | list | None] = redis_kwargs.get("startup_nodes", None) or get_secret( # type: ignore
_startup_nodes: Final[str | list | None] = redis_kwargs.get("startup_nodes", None) or get_secret(
"REDIS_CLUSTER_NODES"
)
@ -381,7 +381,7 @@ def _get_redis_client_logic(**env_overrides):
elif _startup_nodes is None:
redis_kwargs.pop("startup_nodes", None)
_sentinel_nodes: Final[str | list | None] = redis_kwargs.get("sentinel_nodes", None) or get_secret( # type: ignore
_sentinel_nodes: Final[str | list | None] = redis_kwargs.get("sentinel_nodes", None) or get_secret(
"REDIS_SENTINEL_NODES"
)
@ -395,9 +395,7 @@ def _get_redis_client_logic(**env_overrides):
if _sentinel_password is not None:
redis_kwargs["sentinel_password"] = _sentinel_password
_service_name: Final[str | None] = redis_kwargs.get("service_name", None) or get_secret( # type: ignore
"REDIS_SERVICE_NAME"
)
_service_name: Final[str | None] = redis_kwargs.get("service_name", None) or get_secret("REDIS_SERVICE_NAME")
if _service_name is not None:
redis_kwargs["service_name"] = _service_name
@ -412,7 +410,7 @@ def _get_redis_client_logic(**env_overrides):
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account # type: ignore[attr-defined]
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
@ -449,7 +447,7 @@ def _get_redis_client_logic(**env_overrides):
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True # type: ignore[attr-defined]
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
@ -481,7 +479,7 @@ def _get_redis_client_logic(**env_overrides):
def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
_redis_cluster_nodes_in_env: Final[str | None] = get_secret("REDIS_CLUSTER_NODES") # type: ignore
_redis_cluster_nodes_in_env: Final[str | None] = get_secret("REDIS_CLUSTER_NODES")
if _redis_cluster_nodes_in_env is not None:
try:
redis_kwargs["startup_nodes"] = json.loads(_redis_cluster_nodes_in_env)
@ -505,7 +503,7 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
new_startup_nodes.append(ClusterNode(**item))
cluster_kwargs.pop("startup_nodes", None)
return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs) # type: ignore
return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs)
def _get_redis_sentinel_connection_kwargs(redis_kwargs: dict) -> dict:
@ -638,7 +636,7 @@ def get_redis_async_client(
# Create async RedisCluster with IAM token as password if available
cluster_client: Final = async_redis.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs, # type: ignore
**cluster_kwargs,
)
return cluster_client

2
litellm/_redis_credential_provider.py
View file

@ -3,7 +3,7 @@ import threading
import time
from typing import Any, Final
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
from redis.credentials import CredentialProvider
# Azure AD scope for Redis Cache for Azure.
AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default"

4
litellm/_service_logger.py
View file

@ -1,6 +1,6 @@
import asyncio
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm._logging import verbose_logger
@ -16,7 +16,7 @@ if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
Span = Union[_Span, Any]
Span = _Span | Any
OTELClass = OpenTelemetry
else:
Span = Any

2
litellm/_uuid.py
View file

@ -4,7 +4,7 @@ Internal unified UUID helper.
Always uses fastuuid for performance.
"""
import fastuuid as _uuid # type: ignore
import fastuuid as _uuid
# Expose a module-like alias so callers can use: uuid.uuid4()
uuid = _uuid

22
litellm/a2a_protocol/__init__.py
View file

@ -55,19 +55,15 @@ from litellm.a2a_protocol.main import (
from litellm.types.agents import LiteLLMSendMessageResponse
__all__ = [
# Client
"A2AClient",
# Functions
"asend_message",
"send_message",
"asend_message_streaming",
"aget_agent_card",
"create_a2a_client",
# Response types
"LiteLLMSendMessageResponse",
# Exceptions
"A2AError",
"A2AConnectionError",
"A2AAgentCardError",
"A2AClient",
"A2AConnectionError",
"A2AError",
"A2ALocalhostURLError",
"LiteLLMSendMessageResponse",
"aget_agent_card",
"asend_message",
"asend_message_streaming",
"create_a2a_client",
"send_message",
]

6
litellm/a2a_protocol/card_resolver.py
View file

@ -18,8 +18,8 @@ AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent-card.json"
PREV_AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent.json"
try:
from a2a.client import A2ACardResolver as _A2ACardResolver # type: ignore[no-redef]
from a2a.utils.constants import ( # type: ignore[no-redef]
from a2a.client import A2ACardResolver as _A2ACardResolver
from a2a.utils.constants import (
AGENT_CARD_WELL_KNOWN_PATH,
PREV_AGENT_CARD_WELL_KNOWN_PATH,
)
@ -102,7 +102,7 @@ def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
return agent_card
class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
class LiteLLMA2ACardResolver(_A2ACardResolver):
"""
Custom A2A card resolver that supports multiple well-known paths.

10
litellm/a2a_protocol/exception_mapping_utils.py
View file

@ -29,9 +29,9 @@ try:
A2A_SDK_AVAILABLE = True
except ImportError:
A2A_SDK_AVAILABLE = False
Client = None # type: ignore[misc, assignment]
ClientConfig = None # type: ignore[misc, assignment]
create_client = None # type: ignore[misc, assignment]
Client = None
ClientConfig = None
create_client = None
class A2AExceptionCheckers:
@ -219,6 +219,6 @@ async def handle_a2a_localhost_retry(
streaming=is_streaming,
),
)
new_client._litellm_httpx_client = httpx_client # type: ignore[attr-defined]
new_client._litellm_agent_card = agent_card # type: ignore[attr-defined]
new_client._litellm_httpx_client = httpx_client
new_client._litellm_agent_card = agent_card
return new_client

180
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -10,7 +10,7 @@ A2A Streaming Events (in order):
4. Status update (kind: "status-update") - Final status "completed" with final=true
"""
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
import litellm
@ -21,6 +21,8 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
)
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
from litellm.interactions.agents.utils import merge_agent_headers
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.utils import ModelResponse
# litellm_params key carrying the authenticated principal (hashed virtual key) so
# A2A provider configs can scope provider-side state (e.g. LangFlow session memory)
@ -45,47 +47,14 @@ class A2ACompletionBridgeHandler:
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
def _build_completion_params(
params: dict[str, Any],
litellm_params: dict[str, Any],
api_base: str | None = None,
agent_extra_headers: dict[str, str] | None = None,
litellm_params: Mapping[str, Any],
api_base: str | None,
agent_extra_headers: Mapping[str, str] | None,
*,
_skip_a2a_provider_routing: bool = False,
) -> dict[str, Any]:
"""
Handle non-streaming A2A request via litellm.acompletion.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
api_base: API base URL from agent_card_params
agent_extra_headers: Per-request headers (from x-a2a-{agent}-* rewrite and
admin extra_headers) to forward on the upstream HTTP call.
Returns:
A2A SendMessageResponse dict
"""
custom_llm_provider = litellm_params.get("custom_llm_provider")
if not _skip_a2a_provider_routing:
a2a_provider_config: Final = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
if a2a_provider_config is not None:
verbose_logger.info("A2A: Using provider config for %s", custom_llm_provider)
return await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
agent_extra_headers=agent_extra_headers,
)
stream: bool,
) -> Mapping[str, Any]:
# Extract message from params
message: Final = params.get("message", {})
@ -93,7 +62,7 @@ class A2ACompletionBridgeHandler:
openai_messages: Final = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
# Get completion params
custom_llm_provider = litellm_params.get("custom_llm_provider")
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
model: Final = litellm_params.get("model", "agent")
# Build full model string if provider specified
@ -103,14 +72,17 @@ class A2ACompletionBridgeHandler:
else:
full_model = model
verbose_logger.info("A2A completion bridge: model=%s, api_base=%s", full_model, api_base)
if stream:
verbose_logger.info("A2A completion bridge streaming: model=%s, api_base=%s", full_model, api_base)
else:
verbose_logger.info("A2A completion bridge: model=%s, api_base=%s", full_model, api_base)
# Build completion params dict
completion_params: Final[dict[str, Any]] = {
"model": full_model,
"messages": openai_messages,
"api_base": api_base,
"stream": False,
"stream": stream,
}
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
litellm_params_to_add: Final = {
@ -134,8 +106,64 @@ class A2ACompletionBridgeHandler:
static_headers=completion_params.get("extra_headers"),
)
return completion_params
@staticmethod
async def _acompletion(completion_params: Mapping[str, Any]) -> ModelResponse | CustomStreamWrapper:
return await litellm.acompletion(**completion_params)
@staticmethod
async def handle_non_streaming(
request_id: str,
params: dict[str, Any],
litellm_params: dict[str, Any],
api_base: str | None = None,
agent_extra_headers: dict[str, str] | None = None,
*,
_skip_a2a_provider_routing: bool = False,
) -> dict[str, object]:
"""
Handle non-streaming A2A request via litellm.acompletion.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
api_base: API base URL from agent_card_params
agent_extra_headers: Per-request headers (from x-a2a-{agent}-* rewrite and
admin extra_headers) to forward on the upstream HTTP call.
Returns:
A2A SendMessageResponse dict
"""
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if not _skip_a2a_provider_routing:
a2a_provider_config: Final = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
if a2a_provider_config is not None:
verbose_logger.info("A2A: Using provider config for %s", custom_llm_provider)
return await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
agent_extra_headers=agent_extra_headers,
)
completion_params: Final = A2ACompletionBridgeHandler._build_completion_params(
params=params,
litellm_params=litellm_params,
api_base=api_base,
agent_extra_headers=agent_extra_headers,
stream=False,
)
# Call litellm.acompletion
response: Final = await litellm.acompletion(**completion_params)
response: Final = await A2ACompletionBridgeHandler._acompletion(completion_params)
# Transform response to A2A format
a2a_response: Final = A2ACompletionBridgeTransformation.openai_response_to_a2a_response(
@ -156,7 +184,7 @@ class A2ACompletionBridgeHandler:
agent_extra_headers: dict[str, str] | None = None,
*,
_skip_a2a_provider_routing: bool = False,
) -> AsyncIterator[dict[str, Any]]:
) -> AsyncIterator[dict[str, object]]:
"""
Handle streaming A2A request via litellm.acompletion with stream=True.
@ -177,7 +205,7 @@ class A2ACompletionBridgeHandler:
Yields:
A2A streaming response events
"""
custom_llm_provider = litellm_params.get("custom_llm_provider")
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if not _skip_a2a_provider_routing:
a2a_provider_config: Final = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
@ -198,60 +226,20 @@ class A2ACompletionBridgeHandler:
return
# Extract message from params
message: Final = params.get("message", {})
# Create streaming context
ctx: Final = A2AStreamingContext(
request_id=request_id,
input_message=message,
input_message=params.get("message", {}),
)
# Transform A2A message to OpenAI format
openai_messages: Final = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
# Get completion params
custom_llm_provider = litellm_params.get("custom_llm_provider")
model: Final = litellm_params.get("model", "agent")
# Build full model string if provider specified
# Skip prepending if model already starts with the provider prefix
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
full_model = f"{custom_llm_provider}/{model}"
else:
full_model = model
verbose_logger.info("A2A completion bridge streaming: model=%s, api_base=%s", full_model, api_base)
# Build completion params dict
completion_params: Final[dict[str, Any]] = {
"model": full_model,
"messages": openai_messages,
"api_base": api_base,
"stream": True,
}
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
litellm_params_to_add: Final = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
}
completion_params.update(litellm_params_to_add)
# Apply forward metadata AFTER the litellm_params merge so the helper
# sees any agent-owner-configured ``extra_body.metadata`` and can keep
# those keys authoritative over the client-supplied A2A metadata.
A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params(
completion_params=completion_params,
a2a_message=message,
completion_params: Final = A2ACompletionBridgeHandler._build_completion_params(
params=params,
litellm_params=litellm_params,
api_base=api_base,
agent_extra_headers=agent_extra_headers,
stream=True,
)
if agent_extra_headers:
completion_params["extra_headers"] = merge_agent_headers(
dynamic_headers=agent_extra_headers,
static_headers=completion_params.get("extra_headers"),
)
# 1. Emit initial task event (kind: "task", status: "submitted")
task_event: Final = A2ACompletionBridgeTransformation.create_task_event(ctx)
yield task_event
@ -266,12 +254,12 @@ class A2ACompletionBridgeHandler:
yield working_event
# Call litellm.acompletion with streaming
response: Final = await litellm.acompletion(**completion_params)
response: Final = await A2ACompletionBridgeHandler._acompletion(completion_params)
# 3. Accumulate content and emit artifact update
accumulated_text = ""
chunk_count = 0
async for chunk in response: # type: ignore[union-attr]
async for chunk in response:
chunk_count += 1
# Extract delta content
@ -312,7 +300,7 @@ async def handle_a2a_completion(
litellm_params: dict[str, Any],
api_base: str | None = None,
agent_extra_headers: dict[str, str] | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""Convenience function for non-streaming A2A completion."""
return await A2ACompletionBridgeHandler.handle_non_streaming(
request_id=request_id,
@ -329,7 +317,7 @@ async def handle_a2a_completion_streaming(
litellm_params: dict[str, Any],
api_base: str | None = None,
agent_extra_headers: dict[str, str] | None = None,
) -> AsyncIterator[dict[str, Any]]:
) -> AsyncIterator[dict[str, object]]:
"""Convenience function for streaming A2A completion."""
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
request_id=request_id,

10
litellm/a2a_protocol/main.py
View file

@ -59,9 +59,9 @@ try:
A2A_SDK_AVAILABLE = True
except ImportError:
Client = None # type: ignore[misc, assignment]
ClientConfig = None # type: ignore[misc, assignment]
create_client = None # type: ignore[misc, assignment]
Client = None
ClientConfig = None
create_client = None
# Import our custom card resolver that supports multiple well-known paths
from litellm.a2a_protocol.card_resolver import (
@ -788,10 +788,10 @@ async def create_a2a_client(
# Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
# the configured httpx client (with this agent's trace-id/auth headers) without
# excavating a2a-sdk private internals.
a2a_client._litellm_httpx_client = httpx_client # type: ignore[attr-defined]
a2a_client._litellm_httpx_client = httpx_client
agent_card: Final = getattr(a2a_client, "_card", None)
if agent_card is not None:
a2a_client._litellm_agent_card = agent_card # type: ignore[attr-defined]
a2a_client._litellm_agent_card = agent_card
verbose_logger.info("A2A client created for %s", base_url)

2
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -153,7 +153,7 @@ class AnthropicExceptionMapping:
# Optionally add request_id if provided and not present
if request_id and "request_id" not in parsed:
parsed["request_id"] = request_id
return parsed # type: ignore
return parsed
# Extract message - use parsed dict if available, otherwise raw string
if parsed is not None:

170
litellm/assistants/main.py
View file

@ -51,9 +51,7 @@ async def aget_assistants(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -61,7 +59,7 @@ async def aget_assistants(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -98,7 +96,7 @@ def get_assistants(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -132,12 +130,12 @@ def get_assistants(
max_retries=optional_params.max_retries,
organization=organization,
client=client,
aget_assistants=aget_assistants, # type: ignore
) # type: ignore
aget_assistants=aget_assistants,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -145,14 +143,14 @@ def get_assistants(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.get_assistants(
api_base=api_base,
@ -162,7 +160,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
client=client,
aget_assistants=aget_assistants, # type: ignore
aget_assistants=aget_assistants,
litellm_params=litellm_params_dict,
)
else:
@ -173,7 +171,7 @@ def get_assistants(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
@ -185,7 +183,7 @@ def get_assistants(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
@ -210,9 +208,7 @@ async def acreate_assistants(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model=model, custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -220,7 +216,7 @@ async def acreate_assistants(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model=model,
@ -267,7 +263,7 @@ def create_assistants(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -318,12 +314,12 @@ def create_assistants(
organization=organization,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants, # type: ignore
) # type: ignore
async_create_assistants=async_create_assistants,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -331,14 +327,14 @@ def create_assistants(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -363,7 +359,7 @@ def create_assistants(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
if response is None:
@ -392,9 +388,7 @@ async def adelete_assistant(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -402,7 +396,7 @@ async def adelete_assistant(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -442,7 +436,7 @@ def delete_assistant(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -472,9 +466,9 @@ def delete_assistant(
async_delete_assistants=async_delete_assistants,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -482,14 +476,14 @@ def delete_assistant(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -541,9 +535,7 @@ async def acreate_thread(custom_llm_provider: Literal["openai", "azure"], **kwar
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -551,7 +543,7 @@ async def acreate_thread(custom_llm_provider: Literal["openai", "azure"], **kwar
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -608,7 +600,7 @@ def create_thread(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -649,7 +641,7 @@ def create_thread(
acreate_thread=acreate_thread,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_key = (
optional_params.api_key
@ -657,16 +649,16 @@ def create_thread(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -692,10 +684,10 @@ def create_thread(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
async def aget_thread(
@ -715,9 +707,7 @@ async def aget_thread(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -725,7 +715,7 @@ async def aget_thread(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -758,7 +748,7 @@ def get_thread(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
api_base: str | None = None
@ -797,9 +787,9 @@ def get_thread(
aget_thread=aget_thread,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -807,14 +797,14 @@ def get_thread(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -839,10 +829,10 @@ def get_thread(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
### MESSAGES ###
@ -879,9 +869,7 @@ async def a_add_message(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -890,7 +878,7 @@ async def a_add_message(
else:
# Call the synchronous function using run_in_executor
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -937,7 +925,7 @@ def add_message(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
api_key: str | None = None
@ -976,9 +964,9 @@ def add_message(
a_add_message=a_add_message,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -986,14 +974,14 @@ def add_message(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.add_message(
thread_id=thread_id,
@ -1016,11 +1004,11 @@ def add_message(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
async def aget_messages(
@ -1046,9 +1034,7 @@ async def aget_messages(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -1057,7 +1043,7 @@ async def aget_messages(
else:
# Call the synchronous function using run_in_executor
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -1090,7 +1076,7 @@ def get_messages(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -1129,9 +1115,9 @@ def get_messages(
aget_messages=aget_messages,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -1139,14 +1125,14 @@ def get_messages(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.get_messages(
thread_id=thread_id,
@ -1168,11 +1154,11 @@ def get_messages(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
### RUNS ###
@ -1182,7 +1168,7 @@ def arun_thread_stream(
**kwargs,
) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]:
kwargs["arun_thread"] = True
return run_thread(stream=True, event_handler=event_handler, **kwargs) # type: ignore
return run_thread(stream=True, event_handler=event_handler, **kwargs)
async def arun_thread(
@ -1222,9 +1208,7 @@ async def arun_thread(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -1233,7 +1217,7 @@ async def arun_thread(
else:
# Call the synchronous function using run_in_executor
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -1249,7 +1233,7 @@ def run_thread_stream(
event_handler: AssistantEventHandler | None = None,
**kwargs,
) -> AssistantStreamManager[AssistantEventHandler]:
return run_thread(stream=True, event_handler=event_handler, **kwargs) # type: ignore
return run_thread(stream=True, event_handler=event_handler, **kwargs)
def run_thread(
@ -1283,7 +1267,7 @@ def run_thread(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -1329,9 +1313,9 @@ def run_thread(
event_handler=event_handler,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -1339,14 +1323,14 @@ def run_thread(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.run_thread(
thread_id=thread_id,
@ -1366,7 +1350,7 @@ def run_thread(
client=client,
arun_thread=arun_thread,
litellm_params=litellm_params_dict,
) # type: ignore
)
else:
raise litellm.exceptions.BadRequestError(
message=f"LiteLLM doesn't support {custom_llm_provider} for 'run_thread'. Only 'openai' is supported.",
@ -1375,7 +1359,7 @@ def run_thread(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response

8
litellm/assistants/utils.py
View file

@ -8,8 +8,8 @@ from ..types.llms.openai import *
def get_optional_params_add_message(
role: str | None,
content: str | List[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
attachments: List[Attachment] | None,
content: str | list[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
attachments: list[Attachment] | None,
metadata: dict | None,
custom_llm_provider: str,
**kwargs,
@ -57,7 +57,7 @@ def get_optional_params_add_message(
optional_params = litellm.AzureOpenAIAssistantsAPIConfig().map_openai_params_create_message_params(
non_default_params=non_default_params, optional_params=optional_params
)
for k in passed_params.keys():
for k in passed_params:
if k not in default_params:
optional_params[k] = passed_params[k]
return optional_params
@ -128,7 +128,7 @@ def get_optional_params_image_gen(
if n is not None:
optional_params["sampleCount"] = int(n)
for k in passed_params.keys():
for k in passed_params:
if k not in default_params:
optional_params[k] = passed_params[k]
return optional_params

6
litellm/batches/batch_utils.py
View file

@ -274,7 +274,7 @@ async def _fetch_batch_output_file_content(
credentials: Final = _extract_file_access_credentials(litellm_params)
file_content_kwargs.update(credentials)
_file_content: Final = await afile_content(**file_content_kwargs) # type: ignore[reportArgumentType]
_file_content: Final = await afile_content(**file_content_kwargs)
return _file_content.content
@ -432,7 +432,11 @@ def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_prov
usage_object=response_body.get("usage", None) or {},
reasoning_content=None,
)
from litellm.responses.utils import ResponseAPILoggingUtils
_usage_dict: Final = response_body.get("usage", None) or {}
if ResponseAPILoggingUtils._is_response_api_usage(_usage_dict):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_usage_dict)
usage: Final[Usage] = Usage(**_usage_dict)
return usage

32
litellm/batches/main.py
View file

@ -104,7 +104,7 @@ def _resolve_timeout(
@client
async def acreate_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
input_file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
metadata: dict[str, str] | None = None,
@ -154,7 +154,7 @@ async def acreate_batch(
@client
def create_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
input_file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
metadata: dict[str, str] | None = None,
@ -287,7 +287,7 @@ def create_batch(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.create_batch(
_is_async=_is_async,
@ -327,7 +327,7 @@ def create_batch(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_batch", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -370,7 +370,7 @@ async def aretrieve_batch(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -436,7 +436,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.retrieve_batch(
_is_async=_is_async,
@ -498,7 +498,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -545,7 +545,7 @@ def retrieve_batch(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -677,7 +677,7 @@ async def alist_batches(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -723,7 +723,7 @@ def list_batches(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -755,7 +755,7 @@ def list_batches(
max_retries=optional_params.max_retries,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
@ -770,7 +770,7 @@ def list_batches(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.list_batches(
_is_async=_is_async,
@ -813,7 +813,7 @@ def list_batches(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -909,7 +909,7 @@ def cancel_batch(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -959,7 +959,7 @@ def cancel_batch(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.cancel_batch(
_is_async=_is_async,
@ -999,7 +999,7 @@ def cancel_batch(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="cancel_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="cancel_batch", url="https://github.com/BerriAI/litellm"),
),
)
return response

4
litellm/caching/base_cache.py
View file

@ -9,12 +9,12 @@ Has 4 methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any

6
litellm/caching/caching.py
View file

@ -534,11 +534,9 @@ class Cache:
if isinstance(cached_response, dict):
pass
else:
cached_response = json.loads(
cached_response # type: ignore
) # Convert string to dictionary
cached_response = json.loads(cached_response) # Convert string to dictionary
except Exception:
cached_response = ast.literal_eval(cached_response) # type: ignore
cached_response = ast.literal_eval(cached_response)
return cached_response
return cached_result

4
litellm/caching/caching_handler.py
View file

@ -242,7 +242,7 @@ class LLMCachingHandler:
or litellm.cache.get_cache_key(**self.request_kwargs)
)
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
cached_result._hidden_params["cache_key"] = cache_key
return CachingHandlerResponse(cached_result=cached_result)
elif (
call_type == CallTypes.aembedding.value
@ -356,7 +356,7 @@ class LLMCachingHandler:
or litellm.cache.get_cache_key(**self.request_kwargs)
)
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
cached_result._hidden_params["cache_key"] = cache_key
return CachingHandlerResponse(cached_result=cached_result)
return CachingHandlerResponse(cached_result=cached_result)

6
litellm/caching/disk_cache.py
View file

@ -1,12 +1,12 @@
import json
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any
@ -44,7 +44,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response) # type: ignore
cached_response = json.loads(original_cached_response)
except Exception:
cached_response = original_cached_response
return cached_response

4
litellm/caching/dual_cache.py
View file

@ -13,7 +13,7 @@ import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from litellm.types.caching import RedisPipelineIncrementOperation
@ -29,7 +29,7 @@ from .redis_cache import RedisCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any

276
litellm/caching/evicted_client_closer.py Normal file
View file

@ -0,0 +1,276 @@
"""
Deferred close of HTTP/SDK clients that the LLM client cache has evicted.
Eviction only drops the cache's reference to a client. Every OpenAI/Azure SDK
client is a reference cycle (each resource namespace holds the client back), so
an evicted client and its pooled TCP connections survive until a generational
collection runs, which under load is thousands of requests later.
Closing at eviction time is not an option: a request that was handed the client
just before it was evicted is still using it, and closing it underneath that
request raises ``RuntimeError: Cannot send a request, as the client has been
closed.``
So an evicted client is closed once two conditions hold. A grace window must
have passed since its eviction, which covers a request that holds the client
but is momentarily not on the wire, and the client must report no connection in
flight. The second condition is what keeps the first honest: a request may run
for ``litellm.request_timeout`` seconds, 6000 by default, and a streaming
response is bounded only by how long the upstream keeps sending, so no deadline
on its own can promise that a request has finished.
Only clients litellm itself created are closed; a client the caller supplied is
left alone because litellm does not own its lifecycle.
A client that closes synchronously is closed from wherever the cache is next
used. One whose close is a coroutine needs the event loop it was evicted on, so
it waits for a call from that loop rather than having work scheduled onto a loop
it does not belong to. Queued clients are therefore bucketed by what it takes to
close them, and each bucket is ordered by deadline, so a reap walks the entries
that are due rather than the whole queue.
The queue holds its clients weakly, so waiting out a grace window never keeps
alive anything the collector would have reclaimed first.
"""
import asyncio
import contextlib
import inspect
import threading
import time
import weakref
from collections import deque
from collections.abc import Awaitable, Callable, Iterator
from dataclasses import dataclass, replace
from typing import Final
from litellm.constants import (
EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
)
_CLOSABLE_ANYWHERE: Final = "closable-anywhere"
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
_BucketKey = str | int
@dataclass(frozen=True, slots=True)
class _PendingClose:
"""A queued close.
The client is held weakly, so queueing one never keeps alive anything the
collector would otherwise have reclaimed first.
``needs_loop`` is set for a client whose close is a coroutine; those can only
be closed from the event loop they were evicted on, recorded in ``loop_id``.
A client that closes synchronously carries neither constraint.
"""
client_ref: "weakref.ref[object]"
loop_id: int | None
needs_loop: bool
close_after: float
def _bucket_key(pending: _PendingClose) -> _BucketKey:
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
if not pending.needs_loop:
return _CLOSABLE_ANYWHERE
if pending.loop_id is None:
return _CLOSABLE_ON_ANY_LOOP
return pending.loop_id
def _running_loop_id() -> int | None:
try:
return id(asyncio.get_running_loop())
except RuntimeError:
return None
def _close_function(client: object) -> Callable[[], object] | None:
close_fn: Final[Callable[[], object] | None] = getattr(client, "aclose", None) or getattr(client, "close", None)
return close_fn
def _transport_of(client: object) -> object:
"""The httpx transport behind an SDK wrapper, a litellm handler, or a bare client."""
for holder in (getattr(client, "_client", None), getattr(client, "client", None), client):
transport: object = getattr(holder, "_transport", None)
if transport is not None:
return transport
return None
def _connection_is_idle(connection: object) -> bool:
"""A pooled connection is idle unless it is servicing a request."""
is_idle: Final[object] = getattr(connection, "is_idle", None)
return bool(is_idle()) if callable(is_idle) else True
def _pool_has_busy_connection(transport: object) -> bool | None:
"""Whether the httpcore pool behind the transport is servicing a request.
``None`` when there is no such pool, so the caller can ask the other backend.
"""
pooled: Final[object] = getattr(getattr(transport, "_pool", None), "connections", None)
if not isinstance(pooled, (list, tuple)):
return None
return any(
not _connection_is_idle(connection) # pyright: ignore[reportUnknownArgumentType] # untyped pool list
for connection in pooled # pyright: ignore[reportUnknownVariableType] # untyped pool list
)
def _has_connection_in_flight(client: object) -> bool:
"""Whether the client is servicing a request right now.
Both connection backends litellm uses already account for the connections
they have handed out, so this reads the client's own lease accounting rather
than inferring it from elapsed time: httpcore reports a non-idle connection
for the whole of a response including a stream, and aiohttp holds the
connection in ``_acquired`` over the same span.
A client that cannot answer is reported as idle, which leaves the grace
window as the only guard, exactly as it was before this check existed.
"""
try:
transport: Final = _transport_of(client)
pooled_busy: Final = _pool_has_busy_connection(transport)
if pooled_busy is not None:
return pooled_busy
session: Final[object] = getattr(transport, "client", None)
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
except Exception: # noqa: BLE001 - a client that cannot report its state is treated as idle
return False
async def _close_quietly(closing: Awaitable[object]) -> None:
with contextlib.suppress(Exception):
await closing
class EvictedClientCloser:
"""Closes evicted, litellm-owned clients once they are idle and out of grace."""
def __init__(
self,
grace_seconds: float = EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
max_pending: int = EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
clock: Callable[[], float] = time.monotonic,
) -> None:
self._grace_seconds = grace_seconds
self._max_pending = max_pending
self._clock = clock
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
self._pending_count = 0
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
def mark_owned(self, client: object) -> None:
"""Record that litellm created this client, so it may be closed on eviction."""
try:
self._owned.add(client)
except TypeError:
pass # values that cannot be weak-referenced are never litellm clients
def _is_owned(self, client: object) -> bool:
try:
return client in self._owned
except TypeError:
return False # unhashable values are never litellm clients
def schedule(self, client: object) -> None:
"""Queue an evicted client for closing once it is idle and out of grace.
Past ``max_pending`` the client is left to the collector instead, so a
workload that churns the cache cannot grow this queue without bound.
Every queued entry comes due within one grace window, so the capacity it
occupies is returned within that window rather than held.
"""
if client is None or not self._is_owned(client):
return
close_fn: Final = _close_function(client)
if close_fn is None:
return
if self._pending_count >= self._max_pending:
return
self._enqueue(
_PendingClose(
client_ref=weakref.ref(client),
loop_id=_running_loop_id(),
needs_loop=inspect.iscoroutinefunction(close_fn),
close_after=self._clock() + self._grace_seconds,
)
)
def reap(self) -> None:
"""Close every queued client that is due, idle, and closable from here.
Called from the cache's read path, so the empty-queue exit comes first and
the work done past it is proportional to what is due, not to the queue.
"""
if not self._pending_count:
return
now: Final = self._clock()
for pending in self._take_due(_running_loop_id(), now):
client = pending.client_ref()
if client is None:
continue
if _has_connection_in_flight(client):
self._enqueue(replace(pending, close_after=now + self._grace_seconds))
continue
self._close(client)
@property
def pending_count(self) -> int:
return self._pending_count
def _enqueue(self, pending: _PendingClose) -> None:
"""Append to the entry's bucket, dropping any dead entries it queues behind.
Deadlines only ever move forward, so appending keeps each bucket ordered
by deadline, and entries whose client the collector already took sit at
the front rather than having to be searched for.
"""
with self._queue_lock:
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque()) # mutable-ok: FIFO by design
while bucket and bucket[0].client_ref() is None:
bucket.popleft()
self._pending_count -= 1
bucket.append(pending)
self._pending_count += 1
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
with self._queue_lock:
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
bucket: Final = self._buckets.get(key)
if bucket is None:
return
while bucket and bucket[0].close_after <= now:
self._pending_count -= 1
yield bucket.popleft()
if not bucket:
del self._buckets[key]
def _close(self, client: object) -> None:
close_fn: Final = _close_function(client)
if close_fn is None:
return
try:
closing: Final = close_fn()
except Exception: # noqa: BLE001 - a discarded client's close must never surface to callers
return
if not inspect.isawaitable(closing):
return
task: Final = asyncio.get_running_loop().create_task(_close_quietly(closing))
self._close_tasks.add(task)
task.add_done_callback(self._close_tasks.discard)
default_evicted_client_closer: Final = EvictedClientCloser()

45
litellm/caching/llm_caching_handler.py
View file

@ -5,21 +5,44 @@ Add the event loop to the cache key, to prevent event loop closed errors.
import asyncio
from typing import Final
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
from .in_memory_cache import InMemoryCache
class LLMClientCache(InMemoryCache):
"""Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
IMPORTANT: This cache intentionally does NOT close clients on eviction.
Evicted clients may still be in use by in-flight requests. Closing them
eagerly causes ``RuntimeError: Cannot send a request, as the client has
been closed.`` errors in production after the TTL (1 hour) expires.
An evicted client is never closed on the spot: a request handed the client
just before eviction is still using it, and closing it there raises
``RuntimeError: Cannot send a request, as the client has been closed.``
Clients that are no longer referenced will be garbage-collected normally.
For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
Nor can eviction be left to rely on garbage collection. The SDK clients are
reference cycles, so an evicted client and its open TCP connections survive
until a generational collection runs. Instead a client litellm created is
handed to ``EvictedClientCloser``, which closes it once a grace window has
passed. Clients the caller supplied are left untouched.
"""
def __init__(
self,
max_size_in_memory: int | None = 200,
default_ttl: int | None = 600,
max_size_per_item: int | None = 1024,
evicted_client_closer: EvictedClientCloser | None = None,
) -> None:
super().__init__(
max_size_in_memory=max_size_in_memory,
default_ttl=default_ttl,
max_size_per_item=max_size_per_item,
)
self.evicted_client_closer = evicted_client_closer or default_evicted_client_closer
def _remove_key(self, key: str) -> None:
evicted: Final[object] = self.cache_dict.get(key)
super()._remove_key(key)
self.evicted_client_closer.schedule(evicted)
self.evicted_client_closer.reap()
def update_cache_key_with_event_loop(self, key):
"""
Add the event loop to the cache key, to prevent event loop closed errors.
@ -32,16 +55,22 @@ class LLMClientCache(InMemoryCache):
except RuntimeError: # handle no current running event loop
return key
def set_cache(self, key, value, **kwargs):
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
if litellm_owned_client:
self.evicted_client_closer.mark_owned(value)
key = self.update_cache_key_with_event_loop(key)
return super().set_cache(key, value, **kwargs)
async def async_set_cache(self, key, value, **kwargs):
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
if litellm_owned_client:
self.evicted_client_closer.mark_owned(value)
key = self.update_cache_key_with_event_loop(key)
return await super().async_set_cache(key, value, **kwargs)
def get_cache(self, key, **kwargs):
key = self.update_cache_key_with_event_loop(key)
self.evicted_client_closer.reap()
return super().get_cache(key, **kwargs)

50
litellm/caching/redis_cache.py
View file

@ -18,7 +18,7 @@ import time
from collections.abc import Awaitable, Callable, Sequence
from contextvars import ContextVar
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final, TypeVar, Union, cast
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
@ -49,7 +49,7 @@ if TYPE_CHECKING:
cluster_pipeline = ClusterPipeline
async_redis_client = Redis
async_redis_cluster_client = RedisCluster
Span = Union[_Span, Any]
Span = _Span | Any
else:
pipeline = Any
cluster_pipeline = Any
@ -242,7 +242,7 @@ async def _run_under_circuit_breaker(
return result
def _redis_circuit_breaker_guard(method): # type: ignore
def _redis_circuit_breaker_guard(method):
"""
Decorator for RedisCache async methods.
Checks the circuit breaker before each call; records success/failure after.
@ -256,7 +256,7 @@ def _redis_circuit_breaker_guard(method): # type: ignore
"""
@functools.wraps(method)
async def wrapper(self, *args, **kwargs): # type: ignore
async def wrapper(self, *args, **kwargs):
return await _run_under_circuit_breaker(
self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
)
@ -319,7 +319,7 @@ class RedisCache(BaseCache):
self.redis_version = "Unknown"
try:
if not coroutine_checker.is_async_callable(self.redis_client):
self.redis_version = self.redis_client.info()["redis_version"] # type: ignore
self.redis_version = self.redis_client.info()["redis_version"]
except Exception:
pass
@ -355,7 +355,7 @@ class RedisCache(BaseCache):
# SYNC HEALTH PING
try:
if hasattr(self.redis_client, "ping"):
self.redis_client.ping() # type: ignore
self.redis_client.ping()
except Exception as e:
verbose_logger.error("Error connecting to Sync Redis client", extra={"error": str(e)})
self._handle_sync_ping_error(e)
@ -423,7 +423,7 @@ class RedisCache(BaseCache):
redis_async_client = get_redis_async_client(connection_pool=self.async_redis_conn_pool, **self.redis_kwargs)
in_memory_llm_clients_cache.set_cache(key=cache_key, value=redis_async_client)
self.redis_async_client = redis_async_client # type: ignore
self.redis_async_client = redis_async_client
return redis_async_client
def check_and_fix_namespace(self, key: str) -> str:
@ -431,7 +431,7 @@ class RedisCache(BaseCache):
Make sure each key starts with the given namespace
"""
if key is None:
return key # type: ignore[return-value]
return key
if self.namespace is not None and not key.startswith(self.namespace):
key = self.namespace + ":" + key
@ -493,7 +493,7 @@ class RedisCache(BaseCache):
key = self.check_and_fix_namespace(key=key)
try:
start_time = time.time()
result: Final[int] = _redis_client.incr(name=key, amount=value) # type: ignore
result: Final[int] = _redis_client.incr(name=key, amount=value)
end_time = time.time()
_duration = end_time - start_time
self.service_logger_obj.service_success_hook(
@ -520,7 +520,7 @@ class RedisCache(BaseCache):
if current_ttl == -1:
# Key has no expiration
start_time = time.time()
_redis_client.expire(key, set_ttl) # type: ignore
_redis_client.expire(key, set_ttl)
end_time = time.time()
_duration = end_time - start_time
self.service_logger_obj.service_success_hook(
@ -555,7 +555,7 @@ class RedisCache(BaseCache):
return []
pattern = self.check_and_fix_namespace(key=pattern)
async for key in _redis_client.scan_iter(match=pattern + "*", count=count): # type: ignore
async for key in _redis_client.scan_iter(match=pattern + "*", count=count):
keys.append(key)
if len(keys) >= count:
break
@ -680,7 +680,7 @@ class RedisCache(BaseCache):
start_time: Final = time.time()
try:
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
except Exception as e:
end_time = time.time()
_duration = end_time - start_time
@ -773,7 +773,7 @@ class RedisCache(BaseCache):
_td: timedelta | None = None
if ttl is not None:
_td = timedelta(seconds=ttl)
pipe.set( # type: ignore
pipe.set(
name=cache_key,
value=json_cache_value,
ex=_td,
@ -849,7 +849,7 @@ class RedisCache(BaseCache):
"""Helper function for async_set_cache_sadd. Separated for testing."""
ttl = self.get_ttl(ttl=ttl)
try:
await redis_client.sadd(key, *value) # type: ignore
await redis_client.sadd(key, *value)
if ttl is not None:
_td: Final = timedelta(seconds=ttl)
await redis_client.expire(key, _td)
@ -862,7 +862,7 @@ class RedisCache(BaseCache):
start_time: Final = time.time()
try:
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
except Exception as e:
end_time = time.time()
_duration = end_time - start_time
@ -945,7 +945,7 @@ class RedisCache(BaseCache):
) -> float:
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
start_time: Final = time.time()
_used_ttl: Final = self.get_ttl(ttl=ttl)
key = self.check_and_fix_namespace(key=key)
@ -1080,7 +1080,7 @@ class RedisCache(BaseCache):
We use a wrapper so RedisCluster can override this method
"""
return self.redis_client.mget(keys=keys) # type: ignore
return self.redis_client.mget(keys=keys)
async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
"""
@ -1089,7 +1089,7 @@ class RedisCache(BaseCache):
We use a wrapper so RedisCluster can override this method
"""
async_redis_client: Final = self.init_async_client()
return await async_redis_client.mget(keys=keys) # type: ignore
return await async_redis_client.mget(keys=keys)
def batch_get_cache(
self,
@ -1147,7 +1147,7 @@ class RedisCache(BaseCache):
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
key = self.check_and_fix_namespace(key=key)
start_time: Final = time.time()
@ -1269,7 +1269,7 @@ class RedisCache(BaseCache):
print_verbose("Pinging Sync Redis Cache")
start_time: Final = time.time()
try:
response: Final[bool] = self.redis_client.ping() # type: ignore
response: Final[bool] = self.redis_client.ping()
print_verbose(f"Redis Cache PING: {response}")
## LOGGING ##
end_time = time.time()
@ -1339,7 +1339,7 @@ class RedisCache(BaseCache):
await _redis_client.delete(*keys)
def client_list(self) -> list:
client_list: Final[list] = self.redis_client.client_list() # type: ignore
client_list: Final[list] = self.redis_client.client_list()
return client_list
def info(self):
@ -1376,10 +1376,10 @@ class RedisCache(BaseCache):
redis_client: Final = redis_async.Redis(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping() # type: ignore[misc]
ping_result: Final = await redis_client.ping()
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]
await redis_client.aclose()
if ping_result:
return {
@ -1448,7 +1448,7 @@ class RedisCache(BaseCache):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
start_time: Final = time.time()
print_verbose(f"Increment Async Redis Cache Pipeline: increment list: {increment_list}")
@ -1769,7 +1769,7 @@ class RedisCache(BaseCache):
or None
)
except Exception:
decoded_results.append(r) # type: ignore
decoded_results.append(r)
else:
decoded_results.append(None)
return decoded_results

14
litellm/caching/redis_cluster_cache.py
View file

@ -5,7 +5,7 @@ Key differences:
- RedisClient NEEDs to be re-used across requests, adds 3000ms latency if it's re-created
"""
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from litellm.caching.redis_cache import RedisCache
@ -16,7 +16,7 @@ if TYPE_CHECKING:
pipeline = Pipeline
async_redis_client = Redis
Span = Union[_Span, Any]
Span = _Span | Any
else:
pipeline = Any
async_redis_client = Any
@ -47,14 +47,14 @@ class RedisClusterCache(RedisCache):
"""
Overrides `_run_redis_mget_operation` in redis_cache.py
"""
return self.redis_client.mget_nonatomic(keys=keys) # type: ignore
return self.redis_client.mget_nonatomic(keys=keys)
async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
"""
Overrides `_async_run_redis_mget_operation` in redis_cache.py
"""
async_redis_cluster_client: Final = self.init_async_client()
return await async_redis_cluster_client.mget_nonatomic(keys=keys) # type: ignore
return await async_redis_cluster_client.mget_nonatomic(keys=keys)
async def test_connection(self) -> dict:
"""
@ -78,14 +78,14 @@ class RedisClusterCache(RedisCache):
# Create a fresh Redis Cluster client with current settings
redis_client: Final = redis_async.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs, # type: ignore
**cluster_kwargs,
)
# Test the connection
ping_result: Final = await redis_client.ping() # type: ignore[attr-defined, misc]
ping_result: Final = await redis_client.ping()
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]
await redis_client.aclose()
if ping_result:
return {

6
litellm/caching/redis_semantic_cache.py
View file

@ -126,8 +126,8 @@ class RedisSemanticCache(BaseCache):
# CustomTextVectorizer probes its embedding dimension at construction by
# embedding "dimension test", so the first cache request issues one extra
# billable embedding on top of the request's own.
from redisvl.extensions.llmcache import SemanticCache # type: ignore[import-not-found, import-untyped]
from redisvl.utils.vectorize import CustomTextVectorizer # type: ignore[import-not-found, import-untyped]
from redisvl.extensions.llmcache import SemanticCache
from redisvl.utils.vectorize import CustomTextVectorizer
try:
cache_vectorizer: Final = CustomTextVectorizer(self._get_embedding)
@ -207,7 +207,7 @@ class RedisSemanticCache(BaseCache):
return {self.CACHE_KEY_FIELD_NAME: str(key)}
def _get_cache_key_filter_expression(self, key: str) -> Any:
from redisvl.query.filter import Tag # type: ignore[import-not-found, import-untyped]
from redisvl.query.filter import Tag
return Tag(self.CACHE_KEY_FIELD_NAME) == str(key)

2
litellm/caching/s3_cache.py
View file

@ -146,7 +146,7 @@ class S3Cache(BaseCache):
)
return cached_response
except botocore.exceptions.ClientError as e: # type: ignore
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "NoSuchKey":
verbose_logger.debug("S3 Cache: The specified key '%s' does not exist in the S3 bucket.", key)
return None

8
litellm/caching/valkey_semantic_cache.py
View file

@ -85,12 +85,8 @@ class ValkeySemanticCache(RedisSemanticCache):
resolved_url = None
if sync_client is None or async_client is None:
resolved_url = redis_url or self._build_valkey_url(host, port, password, ssl)
self.sync_client = (
sync_client if sync_client is not None else Redis.from_url(resolved_url) # type: ignore[arg-type]
)
self.async_client = (
async_client if async_client is not None else AsyncRedis.from_url(resolved_url) # type: ignore[arg-type]
)
self.sync_client = sync_client if sync_client is not None else Redis.from_url(resolved_url)
self.async_client = async_client if async_client is not None else AsyncRedis.from_url(resolved_url)
print_verbose(f"Valkey semantic-cache initializing index - {self.index_name}")

4
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -238,7 +238,7 @@ class ResponsesToCompletionBridgeHandler:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
streaming_response=result,
sync_stream=True,
json_mode=kwargs.get("json_mode"),
)
@ -336,7 +336,7 @@ class ResponsesToCompletionBridgeHandler:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
streaming_response=result,
sync_stream=False,
json_mode=kwargs.get("json_mode"),
)

34
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -63,9 +63,9 @@ def _get_reasoning_items(
msg: "AllMessageValues",
) -> list[ChatCompletionReasoningItem]:
"""Extract reasoning_items from a message dict with proper typing."""
items: Final = msg.get("reasoning_items") # type: ignore[union-attr]
items: Final = msg.get("reasoning_items")
if items:
return items # type: ignore[return-value]
return items
return []
@ -261,8 +261,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content, # type: ignore[arg-type]
role, # type: ignore
content,
role,
),
}
)
@ -336,7 +336,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
{
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(content, cast(str, role)), # type: ignore[arg-type]
"content": self._convert_content_to_responses_format(content, cast(str, role)),
}
)
@ -360,17 +360,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif key == "response_format":
text_format = self._transform_response_format_to_text_format(value)
if text_format:
responses_api_request["text"] = text_format # type: ignore
responses_api_request["text"] = text_format
elif key == "tool_choice":
responses_api_request["tool_choice"] = ( # type: ignore[assignment]
self._normalize_tool_choice_for_responses_api(value)
)
responses_api_request["tool_choice"] = self._normalize_tool_choice_for_responses_api(value)
elif key == "stream_options":
stream_options = normalize_responses_api_stream_options(value)
if stream_options is not None:
responses_api_request["stream_options"] = stream_options
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key in ResponsesAPIOptionalRequestParams.__annotations__:
responses_api_request[key] = value
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
@ -524,7 +522,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
ResponseApplyPatchToolCall,
)
except ImportError:
ResponseApplyPatchToolCall = None # type: ignore[assignment,misc]
ResponseApplyPatchToolCall = None
from litellm.types.utils import Choices, Message
@ -942,7 +940,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
flat_custom["format"] = convert_custom_tool_format_to_responses_shape(custom_payload["format"])
responses_tools.append(flat_custom)
else:
responses_tools.append(tool) # type: ignore
responses_tools.append(tool)
return cast(list["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
@ -978,7 +976,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _map_reasoning_effort(self, reasoning_effort: str | dict[str, Any]) -> Reasoning | None:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(**reasoning_effort) # type: ignore[typeddict-item]
return Reasoning(**reasoning_effort)
# Check if auto-summary is enabled via flag or environment variable
# Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
@ -988,11 +986,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# If string is passed, map with optional summary based on flag/env var
if reasoning_effort == "none":
return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none") # type: ignore
return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
elif reasoning_effort == "high":
return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
elif reasoning_effort == "xhigh":
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh") # type: ignore[typeddict-item]
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
elif reasoning_effort == "medium":
return (
Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
@ -1108,7 +1106,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
verbose_logger.debug("Skipping unsupported annotation type: %s", type(annotation))
continue
result.append(annotation_dict) # type: ignore
result.append(annotation_dict)
except Exception as e:
# Skip malformed annotations
verbose_logger.debug("Skipping malformed annotation: %s, error: %s", annotation, e)
@ -1254,7 +1252,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
function=function_chunk,
)
if provider_specific_fields:
tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore
tool_call_chunk.provider_specific_fields = provider_specific_fields
return ModelResponseStream(
choices=[

2
litellm/compression/scoring/bm25.py
View file

@ -84,7 +84,7 @@ def bm25_score_messages(
# document tokens that start with that term (min 4 chars match). This lets
# "cook" match "cooking" and "auth" match "authentication" without a full
# stemmer dependency.
def _expand_tf(query_term: str, tf_counts: Counter) -> int: # type: ignore[type-arg]
def _expand_tf(query_term: str, tf_counts: Counter) -> int:
"""Sum TF across all doc tokens that are prefixed by query_term."""
exact: Final = tf_counts.get(query_term, 0)
if exact:

10
litellm/constants.py
View file

@ -197,6 +197,16 @@ RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10
########## Networking constants ##############################################################
_DEFAULT_TTL_FOR_HTTPX_CLIENTS: Final = 3600 # 1 hour, re-use the same httpx client for 1 hour
# The earliest an evicted, litellm-created client may be closed. A request handed the
# client just before eviction is still using it, so nothing is closed inside this window;
# past it, the client is closed once it reports no connection in flight.
EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS: Final = 900
# How many evicted clients may be queued for closing at once. Past this, an evicted client
# is left to the collector rather than letting a cache-churning workload grow the queue
# without bound. Each queued entry is ~100 bytes and comes due within one grace window.
EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000
# Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
# Set to 0 for unlimited (not recommended for production)
AIOHTTP_CONNECTOR_LIMIT: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000))

14
litellm/containers/__init__.py
View file

@ -23,22 +23,20 @@ from .main import (
)
__all__ = [
# Core container operations
"acreate_container",
"adelete_container",
"alist_containers",
"aretrieve_container",
"create_container",
"delete_container",
"list_containers",
"retrieve_container",
# Container file operations (auto-generated from endpoints.json)
"adelete_container_file",
"alist_container_files",
"alist_containers",
"aretrieve_container",
"aretrieve_container_file",
"aretrieve_container_file_content",
"create_container",
"delete_container",
"delete_container_file",
"list_container_files",
"list_containers",
"retrieve_container",
"retrieve_container_file",
"retrieve_container_file_content",
]

12
litellm/containers/main.py
View file

@ -187,7 +187,7 @@ def create_container(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -405,7 +405,7 @@ def list_containers(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -596,7 +596,7 @@ def retrieve_container(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -811,7 +811,7 @@ def delete_container(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -1040,7 +1040,7 @@ def list_container_files(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -1291,7 +1291,7 @@ def upload_container_file(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True

6
litellm/containers/utils.py
View file

@ -53,7 +53,7 @@ class ContainerRequestUtils:
for param in valid_params:
if param in passed_params and passed_params[param] is not None:
container_create_optional_params[param] = passed_params[param] # type: ignore
container_create_optional_params[param] = passed_params[param]
return container_create_optional_params
@ -69,7 +69,7 @@ class ContainerRequestUtils:
filtered_params: Final = {k: v for k, v in container_create_optional_params.items() if k in supported_params}
return container_provider_config.map_openai_params(
container_create_optional_params=filtered_params, # type: ignore
container_create_optional_params=filtered_params,
drop_params=False,
)
@ -90,7 +90,7 @@ class ContainerRequestUtils:
for param in valid_params:
if param in passed_params and passed_params[param] is not None:
container_list_optional_params[param] = passed_params[param] # type: ignore
container_list_optional_params[param] = passed_params[param]
return container_list_optional_params

14
litellm/cost_calculator.py
View file

@ -329,7 +329,7 @@ def cost_per_token(
response: Any | None = None,
### REQUEST MODEL ###
request_model: str | None = None, # original request model for router detection
) -> tuple[float, float]: # type: ignore
) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -878,6 +878,8 @@ def _get_usage_object(
return None
if isinstance(usage_obj, Usage):
return usage_obj
elif isinstance(usage_obj, dict) and litellm.AnthropicConfig.is_anthropic_usage_object(usage_obj):
return litellm.AnthropicConfig().calculate_usage(usage_object=usage_obj, reasoning_content=None)
elif (
usage_obj is not None
and (isinstance(usage_obj, dict) or isinstance(usage_obj, ResponseAPIUsage))
@ -1249,7 +1251,13 @@ def completion_cost(
else:
_usage = usage_obj
if ResponseAPILoggingUtils._is_response_api_usage(_usage):
if litellm.AnthropicConfig.is_anthropic_usage_object(_usage):
_usage = (
litellm.AnthropicConfig()
.calculate_usage(usage_object=_usage, reasoning_content=None)
.model_dump()
)
elif ResponseAPILoggingUtils._is_response_api_usage(_usage):
_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
_usage
).model_dump()
@ -1514,7 +1522,7 @@ def completion_cost(
# see https://replicate.com/pricing
elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost:
# for unmapped replicate model, default to replicate's time tracking logic
return get_replicate_completion_pricing(completion_response, total_time) # type: ignore
return get_replicate_completion_pricing(completion_response, total_time)
if model is None:
raise ValueError(

88
litellm/evals/main.py
View file

@ -141,7 +141,7 @@ def create_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acreate_eval", False) is True
@ -153,7 +153,7 @@ def create_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -162,15 +162,15 @@ def create_eval(
# Build create request
create_request: Final[CreateEvalRequest] = {
"data_source_config": data_source_config, # type: ignore
"testing_criteria": testing_criteria, # type: ignore
"data_source_config": data_source_config,
"testing_criteria": testing_criteria,
}
if name is not None:
create_request["name"] = name
# Merge extra_body if provided
if extra_body:
create_request.update(extra_body) # type: ignore
create_request.update(extra_body)
# Validate environment and get headers
headers = extra_headers or {}
@ -199,7 +199,7 @@ def create_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.create_eval_handler( # type: ignore
response: Final = base_llm_http_handler.create_eval_handler(
url=url,
request_body=request_body,
evals_api_provider_config=evals_api_provider_config,
@ -326,7 +326,7 @@ def list_evals(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("alist_evals", False) is True
@ -338,7 +338,7 @@ def list_evals(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -354,13 +354,13 @@ def list_evals(
if before is not None:
list_params["before"] = before
if order is not None:
list_params["order"] = order # type: ignore
list_params["order"] = order
if order_by is not None:
list_params["order_by"] = order_by # type: ignore
list_params["order_by"] = order_by
# Merge extra_query if provided
if extra_query:
list_params.update(extra_query) # type: ignore
list_params.update(extra_query)
# Validate environment and get headers
headers = extra_headers or {}
@ -385,7 +385,7 @@ def list_evals(
)
# Make HTTP request
response: Final = base_llm_http_handler.list_evals_handler( # type: ignore
response: Final = base_llm_http_handler.list_evals_handler(
url=url,
query_params=query_params,
evals_api_provider_config=evals_api_provider_config,
@ -492,7 +492,7 @@ def get_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aget_eval", False) is True
@ -504,7 +504,7 @@ def get_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -536,7 +536,7 @@ def get_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.get_eval_handler( # type: ignore
response: Final = base_llm_http_handler.get_eval_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -657,7 +657,7 @@ def update_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aupdate_eval", False) is True
@ -669,7 +669,7 @@ def update_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -723,7 +723,7 @@ def update_eval(
# Merge extra_body if provided
if extra_body:
update_request.update(extra_body) # type: ignore
update_request.update(extra_body)
# Validate environment and get headers
headers = extra_headers or {}
@ -755,7 +755,7 @@ def update_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.update_eval_handler( # type: ignore
response: Final = base_llm_http_handler.update_eval_handler(
url=url,
request_body=request_body,
evals_api_provider_config=evals_api_provider_config,
@ -862,7 +862,7 @@ def delete_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("adelete_eval", False) is True
@ -874,7 +874,7 @@ def delete_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -906,7 +906,7 @@ def delete_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.delete_eval_handler( # type: ignore
response: Final = base_llm_http_handler.delete_eval_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1012,7 +1012,7 @@ def cancel_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acancel_eval", False) is True
@ -1024,7 +1024,7 @@ def cancel_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1060,7 +1060,7 @@ def cancel_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.cancel_eval_handler( # type: ignore
response: Final = base_llm_http_handler.cancel_eval_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1191,7 +1191,7 @@ def create_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acreate_run", False) is True
@ -1203,7 +1203,7 @@ def create_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1212,7 +1212,7 @@ def create_run(
# Build create request
create_request: Final[CreateRunRequest] = {
"data_source": data_source, # type: ignore
"data_source": data_source,
}
if name is not None:
create_request["name"] = name
@ -1221,7 +1221,7 @@ def create_run(
# Merge extra_body if provided
if extra_body:
create_request.update(extra_body) # type: ignore
create_request.update(extra_body)
# Validate environment and get headers
headers = extra_headers or {}
@ -1248,7 +1248,7 @@ def create_run(
)
# Make HTTP request (default 600s timeout for long-running operations)
response: Final = base_llm_http_handler.create_run_handler( # type: ignore
response: Final = base_llm_http_handler.create_run_handler(
url=url,
request_body=request_body,
evals_api_provider_config=evals_api_provider_config,
@ -1375,7 +1375,7 @@ def list_runs(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("alist_runs", False) is True
@ -1387,7 +1387,7 @@ def list_runs(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1403,11 +1403,11 @@ def list_runs(
if before is not None:
list_params["before"] = before
if order is not None:
list_params["order"] = order # type: ignore
list_params["order"] = order
# Merge extra_query if provided
if extra_query:
list_params.update(extra_query) # type: ignore
list_params.update(extra_query)
# Validate environment and get headers
headers = extra_headers or {}
@ -1433,7 +1433,7 @@ def list_runs(
)
# Make HTTP request
response: Final = base_llm_http_handler.list_runs_handler( # type: ignore
response: Final = base_llm_http_handler.list_runs_handler(
url=url,
query_params=query_params,
evals_api_provider_config=evals_api_provider_config,
@ -1545,7 +1545,7 @@ def get_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aget_run", False) is True
@ -1557,7 +1557,7 @@ def get_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1590,7 +1590,7 @@ def get_run(
)
# Make HTTP request
response: Final = base_llm_http_handler.get_run_handler( # type: ignore
response: Final = base_llm_http_handler.get_run_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1701,7 +1701,7 @@ def cancel_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acancel_run", False) is True
@ -1713,7 +1713,7 @@ def cancel_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1750,7 +1750,7 @@ def cancel_run(
)
# Make HTTP request
response: Final = base_llm_http_handler.cancel_run_handler( # type: ignore
response: Final = base_llm_http_handler.cancel_run_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1866,7 +1866,7 @@ def delete_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("adelete_run", False) is True
@ -1878,7 +1878,7 @@ def delete_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1915,7 +1915,7 @@ def delete_run(
)
# Make HTTP request
response: Final = base_llm_http_handler.delete_run_handler( # type: ignore
response: Final = base_llm_http_handler.delete_run_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,

54
litellm/exceptions.py
View file

@ -126,7 +126,7 @@ def _get_minimal_error_response() -> httpx.Response:
return _MINIMAL_ERROR_RESPONSE
class AuthenticationError(openai.AuthenticationError): # type: ignore
class AuthenticationError(openai.AuthenticationError):
def __init__(
self,
message,
@ -170,7 +170,7 @@ class AuthenticationError(openai.AuthenticationError): # type: ignore
# raise when invalid models passed, example gpt-8
class NotFoundError(openai.NotFoundError): # type: ignore
class NotFoundError(openai.NotFoundError):
def __init__(
self,
message,
@ -213,7 +213,7 @@ class NotFoundError(openai.NotFoundError): # type: ignore
return _message
class BadRequestError(openai.BadRequestError): # type: ignore
class BadRequestError(openai.BadRequestError):
def __init__(
self,
message,
@ -288,7 +288,7 @@ class ImageFetchError(BadRequestError):
)
class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
message,
@ -327,7 +327,7 @@ class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore
return _message
class Timeout(openai.APITimeoutError): # type: ignore
class Timeout(openai.APITimeoutError):
def __init__(
self,
message,
@ -371,7 +371,7 @@ class Timeout(openai.APITimeoutError): # type: ignore
return _message
class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
class PermissionDeniedError(openai.PermissionDeniedError):
def __init__(
self,
message,
@ -410,7 +410,7 @@ class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
return _message
class RateLimitError(openai.RateLimitError): # type: ignore
class RateLimitError(openai.RateLimitError):
"""
Unified rate-limit error.
@ -501,7 +501,7 @@ class RateLimitError(openai.RateLimitError): # type: ignore
# sub class of rate limit error - meant to give more granularity for error handling context window exceeded errors
class ContextWindowExceededError(BadRequestError): # type: ignore
class ContextWindowExceededError(BadRequestError):
def __init__(
self,
message,
@ -516,8 +516,8 @@ class ContextWindowExceededError(BadRequestError): # type: ignore
self.litellm_debug_info = litellm_debug_info
super().__init__(
message=message,
model=self.model, # type: ignore
llm_provider=self.llm_provider, # type: ignore
model=self.model,
llm_provider=self.llm_provider,
response=response,
litellm_debug_info=self.litellm_debug_info,
) # Call the base class constructor with the parameters it needs
@ -543,7 +543,7 @@ class ContextWindowExceededError(BadRequestError): # type: ignore
# sub class of bad request error - meant to help us catch guardrails-related errors on proxy.
class RejectedRequestError(BadRequestError): # type: ignore
class RejectedRequestError(BadRequestError):
def __init__(
self,
message,
@ -562,8 +562,8 @@ class RejectedRequestError(BadRequestError): # type: ignore
response: Final = httpx.Response(status_code=400, request=request)
super().__init__(
message=self.message,
model=self.model, # type: ignore
llm_provider=self.llm_provider, # type: ignore
model=self.model,
llm_provider=self.llm_provider,
response=response,
litellm_debug_info=self.litellm_debug_info,
) # Call the base class constructor with the parameters it needs
@ -585,7 +585,7 @@ class RejectedRequestError(BadRequestError): # type: ignore
return _message
class ContentPolicyViolationError(BadRequestError): # type: ignore
class ContentPolicyViolationError(BadRequestError):
# Error code: 400 - {'error': {'code': 'content_policy_violation', 'message': 'Your request was rejected as a result of our safety system. Image descriptions generated from your prompt may contain text that is not allowed by our safety system. If you believe this was done in error, your request may succeed if retried, or by adjusting your prompt.', 'param': None, 'type': 'invalid_request_error'}}
def __init__(
self,
@ -605,8 +605,8 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore
self.provider_specific_fields = provider_specific_fields
super().__init__(
message=self.message,
model=self.model, # type: ignore
llm_provider=self.llm_provider, # type: ignore
model=self.model,
llm_provider=self.llm_provider,
response=response,
litellm_debug_info=self.litellm_debug_info,
body=body,
@ -630,7 +630,7 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore
return _message
class ServiceUnavailableError(openai.APIStatusError): # type: ignore
class ServiceUnavailableError(openai.APIStatusError):
def __init__(
self,
message,
@ -678,7 +678,7 @@ class ServiceUnavailableError(openai.APIStatusError): # type: ignore
return _message
class BadGatewayError(openai.APIStatusError): # type: ignore
class BadGatewayError(openai.APIStatusError):
def __init__(
self,
message,
@ -726,7 +726,7 @@ class BadGatewayError(openai.APIStatusError): # type: ignore
return _message
class InternalServerError(openai.InternalServerError): # type: ignore
class InternalServerError(openai.InternalServerError):
def __init__(
self,
message,
@ -775,7 +775,7 @@ class InternalServerError(openai.InternalServerError): # type: ignore
# raise this when the API returns an invalid response object - https://github.com/openai/openai-python/blob/1be14ee34a0f8e42d3f9aa5451aa4cb161f1781f/openai/api_requestor.py#L401
class APIError(openai.APIError): # type: ignore
class APIError(openai.APIError):
def __init__(
self,
status_code: int,
@ -796,7 +796,7 @@ class APIError(openai.APIError): # type: ignore
self.num_retries = num_retries
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=None) # type: ignore
super().__init__(self.message, request=request, body=None)
def __str__(self):
_message = self.message
@ -816,7 +816,7 @@ class APIError(openai.APIError): # type: ignore
# raised if an invalid request (not get, delete, put, post) is made
class APIConnectionError(openai.APIConnectionError): # type: ignore
class APIConnectionError(openai.APIConnectionError):
def __init__(
self,
message,
@ -855,7 +855,7 @@ class APIConnectionError(openai.APIConnectionError): # type: ignore
# raised if an invalid request (not get, delete, put, post) is made
class APIResponseValidationError(openai.APIResponseValidationError): # type: ignore
class APIResponseValidationError(openai.APIResponseValidationError):
def __init__(
self,
message,
@ -902,7 +902,7 @@ class JSONSchemaValidationError(APIResponseValidationError):
super().__init__(model=model, message=message, llm_provider=llm_provider)
class OpenAIError(openai.OpenAIError): # type: ignore
class OpenAIError(openai.OpenAIError):
def __init__(self, original_exception=None):
super().__init__()
self.llm_provider = "openai"
@ -987,7 +987,7 @@ class BudgetExceededError(Exception):
## DEPRECATED ##
class InvalidRequestError(openai.BadRequestError): # type: ignore
class InvalidRequestError(openai.BadRequestError):
def __init__(self, message, model, llm_provider):
self.status_code = 400
self.message = message
@ -1024,7 +1024,7 @@ class MockException(openai.APIError):
self.num_retries = num_retries
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=None) # type: ignore
super().__init__(self.message, request=request, body=None)
class LiteLLMUnknownProvider(BadRequestError):
@ -1070,7 +1070,7 @@ class BlockedPiiEntityError(Exception):
super().__init__(self.message)
class MidStreamFallbackError(ServiceUnavailableError): # type: ignore
class MidStreamFallbackError(ServiceUnavailableError):
def __init__(
self,
message: str,

2
litellm/experimental_mcp_client/client.py
View file

@ -15,7 +15,7 @@ from mcp.client.stdio import stdio_client
streamable_http_client: Any | None = None
try:
import mcp.client.streamable_http as streamable_http_module # type: ignore
import mcp.client.streamable_http as streamable_http_module
streamable_http_client = getattr(streamable_http_module, "streamable_http_client", None)
except ImportError:

32
litellm/files/main.py
View file

@ -131,7 +131,7 @@ async def acreate_file(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -176,7 +176,7 @@ def create_file(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -252,7 +252,7 @@ def create_file(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_file", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_file", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -328,7 +328,7 @@ def file_retrieve(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -419,7 +419,7 @@ def file_retrieve(
request=httpx.Request(
method="create_thread",
url="https://github.com/BerriAI/litellm",
), # type: ignore
),
),
)
@ -465,9 +465,9 @@ async def afile_delete(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return cast(FileDeleted, response) # type: ignore
return cast(FileDeleted, response)
except Exception as e:
raise e
@ -511,7 +511,7 @@ def file_delete(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
_is_async: Final = kwargs.pop("is_async", False) is True
@ -596,7 +596,7 @@ def file_delete(
request=httpx.Request(
method="create_thread",
url="https://github.com/BerriAI/litellm",
), # type: ignore
),
),
)
return cast(FileDeleted, response)
@ -639,7 +639,7 @@ async def afile_list(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -673,7 +673,7 @@ def file_list(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -755,7 +755,7 @@ def file_list(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="file_list", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="file_list", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -803,7 +803,7 @@ async def afile_content(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -857,7 +857,7 @@ def file_content(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -987,7 +987,7 @@ def file_content(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -1065,7 +1065,7 @@ def file_content_streaming(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)

6
litellm/files/streaming.py
View file

@ -93,9 +93,9 @@ class FileContentStreamingResponse:
# are released promptly on client disconnects.
with anyio.CancelScope(shield=True):
if hasattr(stream_to_close, "aclose"):
await cast(AsyncIterator[bytes], stream_to_close).aclose() # type: ignore[attr-defined]
await cast(AsyncIterator[bytes], stream_to_close).aclose()
elif hasattr(stream_to_close, "close"):
result: Final = cast(Iterator[bytes], stream_to_close).close() # type: ignore[attr-defined]
result: Final = cast(Iterator[bytes], stream_to_close).close()
if result is not None:
await result
@ -109,7 +109,7 @@ class FileContentStreamingResponse:
self.stream_iterator = cast(Iterator[bytes] | AsyncIterator[bytes], iter(()))
if hasattr(stream_to_close, "close"):
cast(Iterator[bytes], stream_to_close).close() # type: ignore[attr-defined]
cast(Iterator[bytes], stream_to_close).close()
def _build_logging_response(self) -> dict[str, str]:
response: Final = {

48
litellm/fine_tuning/main.py
View file

@ -80,7 +80,7 @@ async def acreate_fine_tuning_job(
hyperparameters: dict | None = {},
suffix: str | None = None,
validation_file: str | None = None,
integrations: List[str] | None = None,
integrations: list[str] | None = None,
seed: int | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: dict[str, str] | None = None,
@ -119,7 +119,7 @@ async def acreate_fine_tuning_job(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -157,7 +157,7 @@ def create_fine_tuning_job(
hyperparameters: dict | None = {},
suffix: str | None = None,
validation_file: str | None = None,
integrations: List[str] | None = None,
integrations: list[str] | None = None,
seed: int | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: dict[str, str] | None = None,
@ -242,9 +242,9 @@ def create_fine_tuning_job(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -252,7 +252,7 @@ def create_fine_tuning_job(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -321,7 +321,7 @@ def create_fine_tuning_job(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -362,7 +362,7 @@ async def acancel_fine_tuning_job(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -396,7 +396,7 @@ def cancel_fine_tuning_job(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -441,7 +441,7 @@ def cancel_fine_tuning_job(
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -449,7 +449,7 @@ def cancel_fine_tuning_job(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -473,7 +473,7 @@ def cancel_fine_tuning_job(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -514,7 +514,7 @@ async def alist_fine_tuning_jobs(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -550,7 +550,7 @@ def list_fine_tuning_jobs(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -594,9 +594,9 @@ def list_fine_tuning_jobs(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -604,7 +604,7 @@ def list_fine_tuning_jobs(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -629,7 +629,7 @@ def list_fine_tuning_jobs(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -669,7 +669,7 @@ async def aretrieve_fine_tuning_job(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -700,7 +700,7 @@ def retrieve_fine_tuning_job(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -733,9 +733,9 @@ def retrieve_fine_tuning_job(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -743,7 +743,7 @@ def retrieve_fine_tuning_job(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -770,7 +770,7 @@ def retrieve_fine_tuning_job(
request=httpx.Request(
method="retrieve_fine_tuning_job",
url="https://github.com/BerriAI/litellm",
), # type: ignore
),
),
)
return response

33
litellm/google_genai/adapters/handler.py
View file

@ -1,7 +1,8 @@
from collections.abc import AsyncIterator, Coroutine
from typing import Any, Final, cast
from collections.abc import AsyncIterator, Coroutine, Mapping
from typing import Final, cast
import litellm
from litellm.types.google_genai.adapters import GenerateContentCompletionKwargs
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse
@ -17,12 +18,12 @@ class GenerateContentToCompletionHandler:
@staticmethod
def _prepare_completion_kwargs(
model: str,
contents: list[dict[str, Any]] | dict[str, Any],
config: dict[str, Any] | None = None,
contents: list[dict[str, object]] | dict[str, object],
config: dict[str, object] | None = None,
stream: bool = False,
litellm_params: GenericLiteLLMParams | None = None,
extra_kwargs: dict[str, Any] | None = None,
) -> dict[str, Any]:
extra_kwargs: Mapping[str, object] | None = None,
) -> GenerateContentCompletionKwargs:
"""Prepare kwargs for litellm.completion/acompletion"""
# Transform generate_content request to completion format
@ -34,7 +35,7 @@ class GenerateContentToCompletionHandler:
**(extra_kwargs or {}),
)
completion_kwargs: Final[dict[str, Any]] = dict(completion_request)
completion_kwargs: Final = dict(completion_request)
# Forward extra_kwargs that should be passed to completion call
if extra_kwargs is not None:
@ -48,17 +49,17 @@ class GenerateContentToCompletionHandler:
if stream:
completion_kwargs["stream"] = stream
return completion_kwargs
return GenerateContentCompletionKwargs(**completion_kwargs)
@staticmethod
async def async_generate_content_handler(
model: str,
contents: list[dict[str, Any]] | dict[str, Any],
contents: list[dict[str, object]] | dict[str, object],
litellm_params: GenericLiteLLMParams,
config: dict[str, Any] | None = None,
config: dict[str, object] | None = None,
stream: bool = False,
**kwargs,
) -> dict[str, Any] | AsyncIterator[bytes]:
**kwargs: object,
) -> dict[str, object] | AsyncIterator[bytes]:
"""Handle generate_content call asynchronously using completion adapter"""
completion_kwargs: Final = GenerateContentToCompletionHandler._prepare_completion_kwargs(
@ -103,13 +104,13 @@ class GenerateContentToCompletionHandler:
@staticmethod
def generate_content_handler(
model: str,
contents: list[dict[str, Any]] | dict[str, Any],
contents: list[dict[str, object]] | dict[str, object],
litellm_params: GenericLiteLLMParams,
config: dict[str, Any] | None = None,
config: dict[str, object] | None = None,
stream: bool = False,
_is_async: bool = False,
**kwargs,
) -> dict[str, Any] | AsyncIterator[bytes] | Coroutine[Any, Any, dict[str, Any] | AsyncIterator[bytes]]:
**kwargs: object,
) -> dict[str, object] | AsyncIterator[bytes] | Coroutine[None, None, dict[str, object] | AsyncIterator[bytes]]:
"""Handle generate_content call using completion adapter"""
if _is_async:

8
litellm/google_genai/main.py
View file

@ -156,7 +156,7 @@ class GenerateContentHelper:
model=model,
custom_llm_provider=custom_llm_provider,
request_body={}, # Will be handled by adapter
generate_content_provider_config=None, # type: ignore
generate_content_provider_config=None,
generate_content_config_dict=dict(config or {}),
native_request_fields={},
litellm_params=litellm_params,
@ -350,7 +350,7 @@ def generate_content(
# Use the adapter to convert to completion format
return GenerateContentToCompletionHandler.generate_content_handler(
model=model,
contents=contents, # type: ignore
contents=contents,
config=setup_result.generate_content_config_dict,
tools=tools,
_is_async=_is_async,
@ -444,7 +444,7 @@ async def agenerate_content_stream(
# Use the adapter to convert to completion format
return await GenerateContentToCompletionHandler.async_generate_content_handler(
model=model,
contents=contents, # type: ignore
contents=contents,
config=setup_result.generate_content_config_dict,
litellm_params=setup_result.litellm_params,
tools=tools,
@ -534,7 +534,7 @@ def generate_content_stream(
# Use the adapter to convert to completion format
return GenerateContentToCompletionHandler.generate_content_handler(
model=model,
contents=contents, # type: ignore
contents=contents,
config=setup_result.generate_content_config_dict,
_is_async=_is_async,
litellm_params=setup_result.litellm_params,

20
litellm/images/main.py
View file

@ -28,7 +28,7 @@ from litellm.utils import exception_type, get_litellm_params
#################### Initialize provider clients ####################
llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
from openai.types.audio.transcription_create_params import FileTypes # type: ignore
from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
@ -112,7 +112,7 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
elif isinstance(init_response, ImageResponse): ## CACHING SCENARIO
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
response = await init_response
if response is None:
raise ValueError("Unable to get Image Response. Please pass a valid llm_provider.")
@ -207,12 +207,12 @@ def image_generation(
aimg_generation: Final = kwargs.get("aimg_generation", False)
litellm_call_id: Final = kwargs.get("litellm_call_id", None)
logger_fn: Final = kwargs.get("logger_fn", None)
mock_response: Final[str | None] = kwargs.get("mock_response", None) # type: ignore
mock_response: Final[str | None] = kwargs.get("mock_response", None)
proxy_server_request: Final = kwargs.get("proxy_server_request", None)
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
model_info: Final = kwargs.get("model_info", None)
metadata: Final = kwargs.get("metadata", {})
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
client: Final = kwargs.get("client", None)
extra_headers: Final = kwargs.get("extra_headers", None)
headers: Final[dict] = kwargs.get("headers", None) or {}
@ -223,7 +223,7 @@ def image_generation(
dynamic_api_key: str | None = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, # type: ignore
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
@ -479,7 +479,7 @@ def image_generation(
elif custom_llm_provider == "bedrock":
if model is None:
raise Exception("Model needs to be set for bedrock")
model_response = bedrock_image_generation.image_generation( # type: ignore
model_response = bedrock_image_generation.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
@ -508,7 +508,7 @@ def image_generation(
async_custom_client = client
## CALL FUNCTION
model_response = custom_handler.aimage_generation( # type: ignore
model_response = custom_handler.aimage_generation(
model=model,
prompt=prompt,
api_key=api_key,
@ -584,7 +584,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
response = await init_response
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
@ -745,7 +745,7 @@ def image_edit(
non_default_params: Final = {
k: v for k, v in kwargs.items() if k not in default_params
} # model-specific params - pass them straight to the model/provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
model_info: Final = kwargs.get("model_info", None)
metadata: Final = kwargs.get("metadata", {})
@ -860,7 +860,7 @@ def image_edit(
if model is None:
raise Exception("Model needs to be set for bedrock")
image_edit_request_params.update(non_default_params)
return bedrock_image_edit.image_edit( # type: ignore
return bedrock_image_edit.image_edit(
model=model,
image=images,
prompt=prompt,

30
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -709,7 +709,7 @@ class SlackAlerting(CustomBatchLogger):
"""Format an alert message for slack"""
headers: Final = {f"{key} Name": key_val, "Provider": provider}
if api_base is not None:
headers["API Base"] = api_base # type: ignore
headers["API Base"] = api_base
headers_str = "\n"
for k, v in headers.items():
@ -767,14 +767,11 @@ class SlackAlerting(CustomBatchLogger):
# Convert deployment_ids back to set if it was stored as a list
if outage_value is not None:
outage_value = self._restore_outage_value_from_cache(outage_value) # type: ignore
outage_value = self._restore_outage_value_from_cache(outage_value)
if (
getattr(exception, "status_code", None) is None
or (
exception.status_code != 408 # type: ignore
and exception.status_code < 500 # type: ignore
)
or (exception.status_code != 408 and exception.status_code < 500)
or self.llm_router is None
):
return
@ -784,7 +781,7 @@ class SlackAlerting(CustomBatchLogger):
_deployment_set.add(deployment_id)
outage_value = ProviderRegionOutageModel(
provider_region_id=cache_key,
alerts=[exception.status_code], # type: ignore
alerts=[exception.status_code],
minor_alert_sent=False,
major_alert_sent=False,
last_updated_at=time.time(),
@ -802,7 +799,7 @@ class SlackAlerting(CustomBatchLogger):
return
if len(outage_value["alerts"]) < self.alerting_args.max_outage_alert_list_size:
outage_value["alerts"].append(exception.status_code) # type: ignore
outage_value["alerts"].append(exception.status_code)
else: # prevent memory leaks
pass
_deployment_set = outage_value["deployment_ids"]
@ -884,13 +881,10 @@ class SlackAlerting(CustomBatchLogger):
max_alerts_size = 10
"""
try:
outage_value: OutageModel | None = await self.internal_usage_cache.async_get_cache(key=deployment_id) # type: ignore
outage_value: OutageModel | None = await self.internal_usage_cache.async_get_cache(key=deployment_id)
if (
getattr(exception, "status_code", None) is None
or (
exception.status_code != 408 # type: ignore
and exception.status_code < 500 # type: ignore
)
or (exception.status_code != 408 and exception.status_code < 500)
or self.llm_router is None
):
return
@ -912,7 +906,7 @@ class SlackAlerting(CustomBatchLogger):
if outage_value is None:
outage_value = OutageModel(
model_id=deployment_id,
alerts=[exception.status_code], # type: ignore
alerts=[exception.status_code],
minor_alert_sent=False,
major_alert_sent=False,
last_updated_at=time.time(),
@ -927,7 +921,7 @@ class SlackAlerting(CustomBatchLogger):
return
if len(outage_value["alerts"]) < self.alerting_args.max_outage_alert_list_size:
outage_value["alerts"].append(exception.status_code) # type: ignore
outage_value["alerts"].append(exception.status_code)
else: # prevent memory leaks
pass
@ -1483,10 +1477,10 @@ Model Info:
if isinstance(response_obj, litellm.ModelResponse) and (
hasattr(response_obj, "usage")
and response_obj.usage is not None # type: ignore
and hasattr(response_obj.usage, "completion_tokens") # type: ignore
and response_obj.usage is not None
and hasattr(response_obj.usage, "completion_tokens")
):
completion_tokens: Final = response_obj.usage.completion_tokens # type: ignore
completion_tokens: Final = response_obj.usage.completion_tokens
if completion_tokens is not None and completion_tokens > 0:
final_value = float(response_s.total_seconds() / completion_tokens)
if isinstance(final_value, timedelta):

4
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -225,11 +225,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# 1. if string, insert cache control in the message
if isinstance(message_content, str):
message["cache_control"] = control # type: ignore
message["cache_control"] = control
# 2. list of objects - only apply to last item per Anthropic spec
elif isinstance(message_content, list):
if len(message_content) > 0 and isinstance(message_content[-1], dict):
message_content[-1]["cache_control"] = control # type: ignore
message_content[-1]["cache_control"] = control
return message
@staticmethod

10
litellm/integrations/argilla.py
View file

@ -10,7 +10,7 @@ import types
from typing import Any, Final
import httpx
from pydantic import BaseModel # type: ignore
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
@ -56,8 +56,8 @@ class ArgillaLogger(CustomBatchLogger):
argilla_base_url=argilla_base_url,
)
self.sampling_rate: float = (
float(os.getenv("ARGILLA_SAMPLING_RATE")) # type: ignore
if os.getenv("ARGILLA_SAMPLING_RATE") is not None and os.getenv("ARGILLA_SAMPLING_RATE").strip().isdigit() # type: ignore
float(os.getenv("ARGILLA_SAMPLING_RATE"))
if os.getenv("ARGILLA_SAMPLING_RATE") is not None and os.getenv("ARGILLA_SAMPLING_RATE").strip().isdigit()
else 1.0
)
@ -196,9 +196,9 @@ class ArgillaLogger(CustomBatchLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate: Final = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
float(os.getenv("LANGSMITH_SAMPLING_RATE"))
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit()
else 1.0
)
random_sample: Final = random.random()

4
litellm/integrations/arize/arize.py
View file

@ -6,7 +6,7 @@ this file has Arize ai specific helper functions
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from litellm.integrations.arize import _utils
from litellm.integrations.arize._utils import ArizeOTELAttributes
@ -21,7 +21,7 @@ if TYPE_CHECKING:
from litellm.types.integrations.arize import Protocol as _Protocol
Protocol = _Protocol
Span = Union[_Span, Any]
Span = _Span | Any
else:
Protocol = Any
Span = Any

10
litellm/integrations/arize/arize_phoenix.py
View file

@ -1,7 +1,7 @@
import os
import threading
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
@ -22,7 +22,7 @@ if TYPE_CHECKING:
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
Span = _Span | Any
OpenTelemetry = _OpenTelemetry
LITELLM_TRACER_NAME: str
else:
@ -40,14 +40,14 @@ else:
)
except ImportError:
LITELLM_TRACER_NAME = "litellm"
OpenTelemetry = None # type: ignore
OpenTelemetry = None
ARIZE_HOSTED_PHOENIX_ENDPOINT: Final = "https://otlp.arize.com/v1/traces"
_MAX_PROJECT_PROVIDERS: Final = 64
class ArizePhoenixLogger(OpenTelemetry): # type: ignore
class ArizePhoenixLogger(OpenTelemetry):
"""
Arize Phoenix logger that sends traces to a Phoenix endpoint.
@ -139,7 +139,7 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
project_attributes["deployment.environment"] = deployment_environment
env_resource: Final = OTELResourceDetector().detect()
project_resource: Final = Resource.create(project_attributes) # type: ignore[arg-type]
project_resource: Final = Resource.create(project_attributes)
return env_resource.merge(project_resource)
def _build_tracer_provider_for_project(self, project_name: str) -> TracerProvider:

4
litellm/integrations/arize/arize_phoenix_prompt_manager.py
View file

@ -174,9 +174,7 @@ class ArizePhoenixTemplateManager:
# Combine rendered content
final_content = " ".join(rendered_content_parts)
rendered_messages.append(
{"role": role, "content": final_content} # type: ignore
)
rendered_messages.append({"role": role, "content": final_content})
return rendered_messages

2
litellm/integrations/bitbucket/__init__.py
View file

@ -27,7 +27,7 @@ def set_global_bitbucket_config(config: dict) -> None:
"""
import litellm
litellm.global_bitbucket_config = config # type: ignore
litellm.global_bitbucket_config = config
def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec") -> "CustomPromptManagement":

14
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -292,9 +292,7 @@ class BitBucketPromptManager(CustomPromptManagement):
final_messages: list[AllMessageValues] = parsed_messages
else:
# If no messages were parsed, prepend the prompt to existing messages
final_messages = [
{"role": "user", "content": rendered_prompt} # type: ignore
] + messages
final_messages = [{"role": "user", "content": rendered_prompt}] + messages
# Update litellm_params with prompt metadata
if litellm_params is None:
@ -345,7 +343,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
} # type: ignore
}
)
current_role = "system"
current_content = [line[7:].strip()] # Remove "System:" prefix
@ -355,7 +353,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
} # type: ignore
}
)
current_role = "user"
current_content = [line[5:].strip()] # Remove "User:" prefix
@ -365,7 +363,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
} # type: ignore
}
)
current_role = "assistant"
current_content = [line[10:].strip()] # Remove "Assistant:" prefix
@ -379,9 +377,9 @@ class BitBucketPromptManager(CustomPromptManagement):
# If no role indicators found, treat as a single user message
if not messages and prompt_content.strip():
messages = [{"role": "user", "content": prompt_content.strip()}] # type: ignore
messages = [{"role": "user", "content": prompt_content.strip()}]
return messages # type: ignore
return messages
def post_call_hook(
self,

6
litellm/integrations/braintrust_logging.py
View file

@ -28,9 +28,9 @@ def get_utc_datetime():
import datetime as dt
if hasattr(dt, "UTC"):
return datetime.now(dt.UTC) # type: ignore
return datetime.now(dt.UTC)
else:
return datetime.utcnow() # type: ignore
return datetime.utcnow()
class BraintrustLogger(CustomLogger):
@ -43,7 +43,7 @@ class BraintrustLogger(CustomLogger):
self.validate_environment(api_key=api_key)
self.api_base = api_base or os.getenv("BRAINTRUST_API_BASE") or API_BASE
self.default_project_id = None
self.api_key: str = api_key or os.getenv("BRAINTRUST_API_KEY") # type: ignore
self.api_key: str = api_key or os.getenv("BRAINTRUST_API_KEY")
self.headers = {
"Authorization": "Bearer " + self.api_key,
"Content-Type": "application/json",

2
litellm/integrations/braintrust_mock_client.py
View file

@ -150,7 +150,7 @@ def create_mock_braintrust_client():
if _original_http_handler_post is None:
_original_http_handler_post = HTTPHandler.post
HTTPHandler.post = _mock_http_handler_post # type: ignore
HTTPHandler.post = _mock_http_handler_post
verbose_logger.debug("[BRAINTRUST MOCK] Patched HTTPHandler.post")
# CRITICAL: Call the factory's initialization function to patch AsyncHTTPHandler.post

323
litellm/integrations/code_interpreter_interception/handler.py
View file

@ -9,13 +9,18 @@ captured stdout back through the typed agentic loop plan.
import json
import time
import uuid
from typing import Any, Final, Literal, TypedDict, cast
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable
from pydantic import ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.base_llm.sandbox.transformation import (
CodeExecutionResult,
ContainerHandle,
)
from litellm.types.integrations.code_interpreter_interception import (
CodeInterpreterInterceptionConfig,
)
@ -37,6 +42,9 @@ from litellm.types.utils import (
ModelResponse,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
LITELLM_CODE_EXECUTION_TOOL_NAME: Final = "litellm_code_execution"
_INTERCEPTION_ACTIVE_KEY: Final = "_code_interpreter_interception_active"
_SANDBOX_KEY: Final = "_code_interpreter_interception_sandbox_key"
@ -109,26 +117,94 @@ class ChatCompletionFunctionToolChoice(TypedDict):
CodeExecutionFunctionToolChoice = ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice
def _extract_session_id(kwargs: dict[str, Any]) -> str | None:
class SandboxToolParams(TypedDict):
sandbox_provider: str
api_key: str | None
api_base: str | None
class SandboxConfigProtocol(Protocol):
async def acreate_sandbox(self) -> ContainerHandle: ...
async def arun_code(self, *, container: ContainerHandle, code: str) -> CodeExecutionResult: ...
async def adelete_sandbox(self, *, container: ContainerHandle) -> object: ...
@runtime_checkable
class _SupportsOutput(Protocol):
output: object
_CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None]
def _output_item_type(item: object) -> object:
if isinstance(item, dict):
item_mapping: Final[dict[str, object]] = item
return item_mapping.get("type")
return getattr(item, "type", None)
def _response_output(response: object) -> object:
if isinstance(response, dict):
response_mapping: Final[Mapping[str, object]] = response
return response_mapping.get("output", [])
return getattr(response, "output", []) or []
def _tool_call_arguments(arguments: object) -> str:
if isinstance(arguments, str):
return arguments
return "" if arguments is None else str(arguments)
def _narrow_tool_call(tool_call: Mapping[str, object]) -> CodeExecutionToolCall:
tool_call_id: Final = tool_call.get("id")
call_id: Final = tool_call.get("call_id")
return {
"id": tool_call_id if isinstance(tool_call_id, str) else None,
"call_id": call_id if isinstance(call_id, str) else None,
"type": "function",
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
"arguments": _tool_call_arguments(tool_call.get("arguments")),
}
def _extract_session_id(kwargs: dict[str, object]) -> str | None:
for meta_key in ("metadata", "litellm_metadata"):
meta = kwargs.get(meta_key)
if isinstance(meta, dict):
sid = meta.get("session_id")
metadata: dict[str, object] = meta
sid = metadata.get("session_id")
if sid and isinstance(sid, str):
return sid
return None
def _extract_identity(kwargs: dict[str, Any]) -> str:
return kwargs.get("user_api_key_hash") or ""
def _extract_identity(kwargs: Mapping[str, object]) -> str:
identity: Final = kwargs.get("user_api_key_hash")
return identity if isinstance(identity, str) else ""
def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None:
def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None:
if sandbox_tool_name is None:
return None
try:
from litellm.sandbox.sandbox_tools import resolve_sandbox_tool
except ImportError:
return None
return resolve_sandbox_tool(sandbox_tool_name)
resolved: Final[dict[str, object] | None] = resolve_sandbox_tool(sandbox_tool_name)
if resolved is None:
return None
provider: Final = resolved.get("sandbox_provider")
api_key: Final = resolved.get("api_key")
api_base: Final = resolved.get("api_base")
return SandboxToolParams(
sandbox_provider=provider if isinstance(provider, str) else "",
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)
class CodeInterpreterInterceptionLogger(CustomLogger):
@ -149,14 +225,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
enabled: bool = True,
enabled_providers: list[str] | None = None,
sandbox_tool_name: str | None = None,
sandbox_config: Any | None = None,
sandbox_config: SandboxConfigProtocol | None = None,
):
super().__init__()
self.enabled = enabled
self.enabled_providers = enabled_providers
self.sandbox_tool_name = sandbox_tool_name
self.sandbox_config = sandbox_config
self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float, str | None]] = {}
self._container_cache: dict[str, _CachedContainer] = {}
@classmethod
def from_config_yaml(cls, config: CodeInterpreterInterceptionConfig) -> "CodeInterpreterInterceptionLogger":
@ -171,19 +247,18 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
litellm_settings: dict[str, Any],
callback_specific_params: dict[str, Any],
) -> "CodeInterpreterInterceptionLogger":
params: CodeInterpreterInterceptionConfig = {}
if "code_interpreter_interception_params" in litellm_settings:
params = litellm_settings["code_interpreter_interception_params"]
elif "code_interpreter_interception" in callback_specific_params and isinstance(
callback_specific_params["code_interpreter_interception"], dict
):
params = cast(
CodeInterpreterInterceptionConfig,
callback_specific_params["code_interpreter_interception"],
)
params: Final[CodeInterpreterInterceptionConfig] = (
litellm_settings["code_interpreter_interception_params"]
if "code_interpreter_interception_params" in litellm_settings
else callback_specific_params["code_interpreter_interception"]
if isinstance(callback_specific_params.get("code_interpreter_interception"), dict)
else {}
)
return CodeInterpreterInterceptionLogger.from_config_yaml(params)
async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None:
async def async_pre_call_deployment_hook(
self, kwargs: dict[str, object], call_type: CallTypes | None
) -> dict | None:
if not kwargs.get("_agentic_loop_depth"):
kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
kwargs.pop(_SANDBOX_KEY, None)
@ -229,13 +304,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
return kwargs
@staticmethod
def _strip_interception_metadata(kwargs: dict[str, Any]) -> None:
def _strip_interception_metadata(kwargs: dict[str, object]) -> None:
metadata: Final = kwargs.get(_LITELLM_METADATA_KEY)
if not isinstance(metadata, dict):
return
current_metadata: Final[dict[str, object]] = metadata
filtered_metadata: Final = {
key: value
for key, value in metadata.items()
for key, value in current_metadata.items()
if not is_interception_internal_key(key)
and not key.startswith("_agentic_loop")
and key != "max_agentic_loops"
@ -247,9 +323,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
kwargs.pop(_LITELLM_METADATA_KEY, None)
@staticmethod
def _write_interception_metadata(kwargs: dict[str, Any]) -> None:
metadata = kwargs.get(_LITELLM_METADATA_KEY)
metadata = dict(metadata) if isinstance(metadata, dict) else {}
def _write_interception_metadata(kwargs: dict[str, object]) -> None:
existing: Final = kwargs.get(_LITELLM_METADATA_KEY)
metadata: Final[dict[str, object]] = dict(existing) if isinstance(existing, dict) else {}
for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _SESSION_SCOPED_KEY, _CONVERTED_STREAM_KEY):
if key in kwargs:
metadata[key] = kwargs[key]
@ -296,20 +372,21 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
}
@staticmethod
def _tool_choice_targets_code_interpreter(tool_choice: Any) -> bool:
def _tool_choice_targets_code_interpreter(tool_choice: object) -> bool:
if not isinstance(tool_choice, dict):
return False
function: Final = tool_choice.get("function")
choice: Final[dict[str, object]] = tool_choice
function: Final = choice.get("function")
return (
tool_choice.get("type") == "code_interpreter"
or tool_choice.get("name") == "code_interpreter"
or tool_choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
choice.get("type") == "code_interpreter"
or choice.get("name") == "code_interpreter"
or choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
or (isinstance(function, dict) and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME)
)
def _resolve_provider(self, kwargs: dict[str, Any]) -> str | None:
def _resolve_provider(self, kwargs: dict[str, object]) -> str | None:
provider: Final = kwargs.get("custom_llm_provider")
if provider:
if isinstance(provider, str) and provider:
return provider
model: Final = kwargs.get("model")
if not isinstance(model, str):
@ -321,7 +398,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
async def async_should_run_agentic_loop(
self,
response: Any,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
@ -351,12 +428,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
tools: dict,
model: str,
messages: list[dict],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: dict,
logging_obj: Any,
response: object,
anthropic_messages_provider_config: object,
anthropic_messages_optional_request_params: dict[str, object],
logging_obj: "LiteLLMLoggingObj",
stream: bool,
kwargs: dict,
kwargs: dict[str, object],
) -> AgenticLoopPlan:
if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE:
return await self._build_chat_completion_agentic_loop_plan(
@ -368,14 +445,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
)
await self._prune_expired_cache()
tool_calls: Final = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
sandbox_key: Final = kwargs.get(_SANDBOX_KEY)
tool_calls: Final = self._agentic_tool_calls(tools)
sandbox_key: Final = self._extract_sandbox_key(kwargs)
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
identity: Final = _extract_identity(kwargs) if is_session else None
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
try:
container_id: Final = cast(str | None, getattr(container, "id", None))
container_id: Final = self._container_id(container)
input_list: Final = self._normalize_messages(messages)
code_interpreter_calls: Final[list[CodeInterpreterCall]] = []
for tool_call in tool_calls:
@ -443,14 +520,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
kwargs: dict[str, object],
) -> AgenticLoopPlan:
await self._prune_expired_cache()
tool_calls: Final = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
sandbox_key: Final = cast(str | None, kwargs.get(_SANDBOX_KEY))
tool_calls: Final = self._agentic_tool_calls(tools)
sandbox_key: Final = self._extract_sandbox_key(kwargs)
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
identity: Final = _extract_identity(cast(dict[str, Any], kwargs)) if is_session else None
identity: Final = _extract_identity(kwargs) if is_session else None
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
try:
container_id: Final = cast(str | None, getattr(container, "id", None))
container_id: Final = self._container_id(container)
tool_results: Final = [
await self._build_chat_completion_tool_result(
container=container,
@ -489,10 +566,28 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
},
)
@staticmethod
def _container_id(container: ContainerHandle) -> str | None:
container_id: Final[object] = getattr(container, "id", None)
return container_id if isinstance(container_id, str) else None
@staticmethod
def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]:
tool_calls: Final = tools.get("tool_calls")
if not isinstance(tool_calls, list):
return []
items: Final[list[object]] = tool_calls
return [_narrow_tool_call(item) for item in items if isinstance(item, dict)]
@staticmethod
def _extract_sandbox_key(kwargs: dict[str, object]) -> str | None:
sandbox_key: Final = kwargs.get(_SANDBOX_KEY)
return sandbox_key if isinstance(sandbox_key, str) else None
async def _build_chat_completion_tool_result(
self,
container: object,
params: dict[str, Any] | None,
container: ContainerHandle,
params: SandboxToolParams | None,
tool_call: CodeExecutionToolCall,
container_id: str | None,
) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
@ -517,10 +612,15 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
)
async def async_agentic_loop_cleanup_hook(self, plan: AgenticLoopPlan, kwargs: dict) -> None:
metadata: Final = plan.metadata or {} if plan else {}
metadata: Final[dict[str, object]] = plan.metadata or {} if plan else {}
if metadata.get("is_session_scoped"):
return
await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
await self._delete_container_for_cache_key(self._metadata_sandbox_key(metadata))
@staticmethod
def _metadata_sandbox_key(metadata: Mapping[str, object]) -> str | None:
sandbox_key: Final = metadata.get("sandbox_key")
return sandbox_key if isinstance(sandbox_key, str) else None
@staticmethod
def _filter_agentic_loop_kwargs(kwargs: dict[str, object]) -> dict[str, object]:
@ -531,12 +631,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
and not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES)
}
def _get_followup_tools(self, tools: object, call_type: CallTypes | None) -> list[dict[str, Any]] | None:
def _get_followup_tools(self, tools: object, call_type: CallTypes | None) -> list[dict[str, object]] | None:
if not isinstance(tools, list):
return None
return [
(
self._get_function_tool(call_type=call_type)
dict(self._get_function_tool(call_type=call_type))
if isinstance(tool, dict) and tool.get("type") == "code_interpreter"
else tool
)
@ -549,34 +649,42 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
k: v for k, v in optional_params.items() if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
}
async def async_post_agentic_loop_response_hook(self, response: Any, plan: AgenticLoopPlan, kwargs: dict) -> Any:
metadata: Final = plan.metadata or {} if plan else {}
async def async_post_agentic_loop_response_hook(
self, response: object, plan: AgenticLoopPlan, kwargs: dict
) -> object:
metadata: Final[dict[str, object]] = plan.metadata or {} if plan else {}
if not metadata.get("is_session_scoped"):
await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
await self._delete_container_for_cache_key(self._metadata_sandbox_key(metadata))
calls: Final = metadata.get("code_interpreter_calls")
if not calls:
if not calls or not isinstance(calls, list):
return response
is_dict: Final = isinstance(response, dict)
output: Final = response.get("output") if is_dict else getattr(response, "output", None)
if not isinstance(output, list):
if isinstance(response, dict):
response_mapping: Final[dict[str, object]] = response
merged_mapping_output: Final = self._merge_code_interpreter_calls(response_mapping.get("output"), calls)
if merged_mapping_output is not None:
response_mapping["output"] = merged_mapping_output
return response
def _item_type(item: Any) -> Any:
return item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
insert_at: Final = next(
(i for i, item in enumerate(output) if _item_type(item) == "message"),
len(output),
)
new_output: Final = output[:insert_at] + list(calls) + output[insert_at:]
if is_dict:
response["output"] = new_output
else:
response.output = new_output
if not isinstance(response, _SupportsOutput):
return response
merged_attr_output: Final = self._merge_code_interpreter_calls(response.output, calls)
if merged_attr_output is not None:
response.output = merged_attr_output
return response
@staticmethod
def _merge_code_interpreter_calls(output: object, calls: Sequence[object]) -> list[object] | None:
if not isinstance(output, list):
return None
items: Final[list[object]] = output
insert_at: Final = next(
(i for i, item in enumerate(items) if _output_item_type(item) == "message"),
len(items),
)
return items[:insert_at] + list(calls) + items[insert_at:]
@staticmethod
def _parse_code(arguments: str) -> str:
try:
@ -584,7 +692,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
except (json.JSONDecodeError, TypeError, AttributeError):
return ""
async def _run_tool_call(self, container: Any, params: dict[str, Any] | None, arguments: str) -> str:
async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str:
try:
code: Final = json.loads(arguments).get("code", "") if arguments else ""
except (json.JSONDecodeError, TypeError):
@ -601,7 +709,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
self,
cache_key: str | None,
identity: str | None = None,
) -> tuple[Any, dict[str, Any] | None]:
) -> tuple[ContainerHandle, SandboxToolParams | None]:
if cache_key:
cached: Final = self._container_cache.get(cache_key)
if cached is not None:
@ -623,7 +731,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
self._container_cache.pop(lru_key, None)
await self._delete_container(container=lru_entry[0], params=lru_entry[1])
async def _create_container(self) -> tuple[Any, dict[str, Any] | None]:
async def _create_container(self) -> tuple[ContainerHandle, SandboxToolParams | None]:
if self.sandbox_config is not None:
return await self.sandbox_config.acreate_sandbox(), None
@ -641,7 +749,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
)
return container, params
async def _run_code(self, container: Any, params: dict[str, Any] | None, code: str) -> Any:
async def _run_code(
self, container: ContainerHandle, params: SandboxToolParams | None, code: str
) -> CodeExecutionResult:
if self.sandbox_config is not None:
return await self.sandbox_config.arun_code(container=container, code=code)
if params is None:
@ -653,7 +763,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
api_key=params.get("api_key"),
)
async def _delete_container(self, container: Any, params: dict[str, Any] | None) -> None:
async def _delete_container(self, container: ContainerHandle, params: SandboxToolParams | None) -> None:
try:
if self.sandbox_config is not None:
await self.sandbox_config.adelete_sandbox(container=container)
@ -677,7 +787,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
return
await self._delete_container(container=cached[0], params=cached[1])
def _normalize_messages(self, messages: Any) -> list[dict[str, Any]]:
def _normalize_messages(self, messages: object) -> list[dict[str, object]]:
if isinstance(messages, str):
return [{"role": "user", "content": messages}]
if isinstance(messages, list):
@ -685,10 +795,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
return []
def _extract_code_execution_tool_calls(self, response: object) -> list[CodeExecutionToolCall]:
if isinstance(response, dict):
output = response.get("output", [])
else:
output = getattr(response, "output", []) or []
output: Final = _response_output(response)
if not isinstance(output, list):
return []
@ -702,9 +809,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
if self._is_code_execution_call(item)
]
def _extract_chat_completion_code_execution_tool_calls(
self, response: ModelResponse | dict[str, Any]
) -> list[CodeExecutionToolCall]:
def _extract_chat_completion_code_execution_tool_calls(self, response: object) -> list[CodeExecutionToolCall]:
model_response: Final = self._to_model_response(response)
if model_response is None:
return []
@ -743,44 +848,46 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
@staticmethod
def _build_chat_completion_assistant_message(
tool_calls: list[CodeExecutionToolCall],
tool_calls: Sequence[CodeExecutionToolCall],
) -> ChatCompletionAssistantMessage:
assistant_tool_calls: Final[list[ChatCompletionAssistantToolCall]] = [
{
"id": tool_call.get("id"),
"type": "function",
"function": {
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
"arguments": tool_call.get("arguments", ""),
},
}
for tool_call in tool_calls
]
return {
"role": "assistant",
"tool_calls": [
cast(
ChatCompletionAssistantToolCall,
{
"id": tool_call.get("id"),
"type": "function",
"function": {
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
"arguments": tool_call.get("arguments", ""),
},
},
)
for tool_call in tool_calls
],
"tool_calls": assistant_tool_calls,
}
@staticmethod
def _to_model_response(
response: ModelResponse | dict[str, Any],
) -> ModelResponse | None:
def _to_model_response(response: object) -> ModelResponse | None:
if isinstance(response, ModelResponse):
return response
if not isinstance(response, dict):
return None
response_fields: Final[dict[str, object]] = response
try:
return ModelResponse(**response)
return ModelResponse(**response_fields)
except (TypeError, ValidationError):
return None
def _is_code_execution_call(self, item: Any) -> bool:
def _is_code_execution_call(self, item: object) -> bool:
if isinstance(item, dict):
return item.get("type") == "function_call" and item.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
return (
getattr(item, "type", None) == "function_call"
and getattr(item, "name", None) == LITELLM_CODE_EXECUTION_TOOL_NAME
)
item_mapping: Final[dict[str, object]] = item
return (
item_mapping.get("type") == "function_call"
and item_mapping.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
)
item_type: Final[object] = getattr(item, "type", None)
item_name: Final[object] = getattr(item, "name", None)
return item_type == "function_call" and item_name == LITELLM_CODE_EXECUTION_TOOL_NAME
async def _prune_expired_cache(self) -> None:
now: Final = time.time()

2
litellm/integrations/compression_interception/handler.py
View file

@ -133,7 +133,7 @@ class CompressionInterceptionLogger(CustomLogger):
self._prune_expired_cache()
compressed: Final = compress( # type: ignore
compressed: Final = compress(
messages=messages,
model=model,
call_type=CallTypes.anthropic_messages,

8
litellm/integrations/custom_guardrail.py
View file

@ -34,7 +34,7 @@ from litellm.types.utils import (
try:
from fastapi.exceptions import HTTPException
except ImportError:
HTTPException = None # type: ignore
HTTPException = None
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -410,7 +410,7 @@ class CustomGuardrail(CustomLogger):
if self.should_route_on_sensitive_data():
try:
self.raise_sensitive_data_route_exception(
route_to_model=self.sensitive_data_route_to_model, # type: ignore
route_to_model=self.sensitive_data_route_to_model,
request_data=request_data,
detection_info=detection_info,
)
@ -892,9 +892,9 @@ class CustomGuardrail(CustomLogger):
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump())) # type: ignore[typeddict-item]
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
else:
guardrail_mode = self.event_hook # type: ignore[assignment]
guardrail_mode = self.event_hook
from litellm.litellm_core_utils.core_helpers import (
filter_exceptions_from_params,

12
litellm/integrations/custom_logger.py
View file

@ -3,7 +3,7 @@
import re
import traceback
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any, Final, Optional, Union
from typing import TYPE_CHECKING, Any, Final, Optional
from pydantic import BaseModel
@ -39,7 +39,7 @@ if TYPE_CHECKING:
)
from litellm.types.router import PreRoutingHookResponse
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any
LiteLLMLoggingObj = Any
@ -783,13 +783,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
- Converting to string and then truncating the logged content catches this
2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
"""
field_value: Final = standard_logging_object.get(field_name) # type: ignore
field_value: Final = standard_logging_object.get(field_name)
if field_value:
str_value: Final = str(field_value)
if len(str_value) > max_length:
standard_logging_object[field_name] = self._truncate_text( # type: ignore
text=str_value, max_length=max_length
)
standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
def _truncate_text(self, text: str, max_length: int) -> str:
"""Truncate text if it exceeds max_length"""
@ -911,7 +909,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
for callback_obj in all_callbacks:
if hasattr(callback_obj, "increment_callback_logging_failure"):
verbose_logger.debug("Incrementing callback failure metric for %s", callback_name)
callback_obj.increment_callback_logging_failure(callback_name=callback_name) # type: ignore
callback_obj.increment_callback_logging_failure(callback_name=callback_name)
return
verbose_logger.debug(

4
litellm/integrations/datadog/datadog.py
View file

@ -500,7 +500,7 @@ class DataDogLogger(
response: Final = self.sync_client.post(
url=self.intake_url,
json=dd_payload, # type: ignore
json=dd_payload,
headers=headers,
)
@ -616,7 +616,7 @@ class DataDogLogger(
response: Final = await self.async_client.post(
url=self.intake_url,
data=compressed_data, # type: ignore
data=compressed_data,
headers=headers,
)
return response

8
litellm/integrations/datadog/datadog_metrics.py
View file

@ -91,9 +91,9 @@ class DatadogMetricsLogger(CustomBatchLogger):
metadata: Final = log.get("metadata", {}) or {}
team_tag: Final = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias") # type: ignore
or metadata.get("team_alias")
or metadata.get("user_api_key_team_id")
or metadata.get("team_id") # type: ignore
or metadata.get("team_id")
)
if team_tag:
@ -193,7 +193,7 @@ class DatadogMetricsLogger(CustomBatchLogger):
# Extract status code from error information
status_code = "500" # default
error_information: Final = standard_logging_object.get("error_information", {}) or {}
error_code: Final = error_information.get("error_code") # type: ignore
error_code: Final = error_information.get("error_code")
if error_code is not None:
status_code = str(error_code)
@ -237,7 +237,7 @@ class DatadogMetricsLogger(CustomBatchLogger):
response: Final = await self.async_client.post(
self.upload_url,
content=compressed_data,
headers=headers, # type: ignore
headers=headers,
)
response.raise_for_status()

2
litellm/integrations/dotprompt/__init__.py
View file

@ -24,7 +24,7 @@ def set_global_prompt_directory(directory: str) -> None:
"""
import litellm
litellm.global_prompt_directory = directory # type: ignore
litellm.global_prompt_directory = directory
def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict:

2
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -311,7 +311,7 @@ class DotpromptManager(CustomPromptManagement):
def _create_message(self, role: str, content: str) -> AllMessageValues:
"""Create a message with the specified role and content."""
return {
"role": role, # type: ignore
"role": role,
"content": content,
}

4
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -31,7 +31,7 @@ class PromptTemplate:
self.output_format = self.metadata.get("output", {}).get("format")
self.output_schema = self.metadata.get("output", {}).get("schema", {})
self.optional_params = {}
for key in self.metadata.keys():
for key in self.metadata:
if key not in restricted_keys:
self.optional_params[key] = self.metadata[key]
@ -253,7 +253,7 @@ class PromptManager:
"dict": dict,
}
return type_mapping.get(schema_type.lower(), str) # type: ignore
return type_mapping.get(schema_type.lower(), str)
def get_prompt(self, prompt_id: str, version: int | None = None) -> PromptTemplate | None:
"""

Some files were not shown because too many files have changed in this diff Show more