scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge branch 'litellm_internal_staging' into litellm_models_endpoint_access_groups

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
milan 2026-08-05 19:51:01 +00:00
commit 16eac1b109
877 changed files with 19656 additions and 12262 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
.circleci/config.yml
View file

@ -88,6 +88,29 @@ commands:
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
install_node:
description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads."
steps:
- run:
name: Install Node.js 24.19.0
command: |
NODE_VERSION="24.19.0"
NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647"
NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)"
if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then
echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2
exit 1
fi
curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c -
sudo mkdir -p /opt/node
sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1
rm -f "/tmp/${NODE_TARBALL}"
echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV"
export PATH="/opt/node/bin:$PATH"
node --version
npm --version
install_rust:
description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
steps:
@ -2594,18 +2617,7 @@ jobs:
# Install Node.js directly from nodejs.org with SHA256 verification,
# instead of piping NodeSource's setup_24.x apt-repo installer into
# sudo bash (which runs a mutable upstream script unattended).
- run:
name: Install Node.js 24.19.0
command: |
NODE_VERSION="24.19.0"
NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647"
curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c -
sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1
rm -f "/tmp/${NODE_TARBALL}"
node --version
npm --version
- install_node
- run:
name: Install Node.js test dependencies
@ -2836,6 +2848,7 @@ jobs:
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- install_node
- install_uv
- install_rust
- restore_cache:
@ -2852,7 +2865,7 @@ jobs:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
- ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
# The cimg/python:3.12-browsers image already ships the Chromium system
@ -2867,7 +2880,7 @@ jobs:
npm ci
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- tests/e2e/ui/node_modules
@ -2979,6 +2992,7 @@ jobs:
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- install_node
- install_uv
- install_rust
- restore_cache:
@ -2995,7 +3009,7 @@ jobs:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
- ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
command: |
@ -3005,7 +3019,7 @@ jobs:
npm ci
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- tests/e2e/ui/node_modules

46
.flake8
View file

@ -1,46 +0,0 @@
[flake8]
ignore =
# The following ignores can be removed when formatting using black
W191,W291,W292,W293,W391,W504
E101,E111,E114,E116,E117,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,
E201,E202,E221,E222,E225,E226,E231,E241,E251,E252,E261,E265,E271,E272,E275,
E301,E302,E303,E305,E306,
# line break before binary operator
W503,
# inline comment should start with '# '
E262,
# too many leading '#' for block comment
E266,
# multiple imports on one line
E401,
# module level import not at top of file
E402,
# Line too long (82 > 79 characters)
E501,
# comparison to None should be 'if cond is None:'
E711,
# comparison to True should be 'if cond is True:' or 'if cond:'
E712,
# do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
E721,
# do not use bare 'except'
E722,
# x is imported but unused
F401,
# 'from . import *' used; unable to detect undefined names
F403,
# x may be undefined, or defined from star imports:
F405,
# f-string is missing placeholders
F541,
# dictionary key '' repeated with different values
F601,
# redefinition of unused x from line 123
F811,
# undefined name x
F821,
# local variable x is assigned to but never used
F841,
# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
extend-ignore = E203

13
.github/workflows/_test-unit-base.yml vendored
View file

@ -154,6 +154,19 @@ jobs:
merge-multiple: true
- name: Upload to Codecov
id: codecov-upload
continue-on-error: true
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false
- name: Upload to Codecov (retry)
if: steps.codecov-upload.outcome == 'failure'
continue-on-error: true
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true

3
.github/workflows/test-linting.yml vendored
View file

@ -104,9 +104,8 @@ jobs:
- name: Check basedpyright budget (delta vs base)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
NODE_OPTIONS: --max-old-space-size=12288
run: |
(uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
- name: Check tests/e2e basedpyright (zero errors)
env:

4
CLAUDE.md
View file

@ -39,10 +39,6 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Python max line length is 120, not 88
On a fresh worktree or clone, run `make bootstrap` before anything else. It provisions everything tests, `make pre-commit`, and a local proxy need
Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit
When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in

8
Makefile
View file

@ -75,7 +75,7 @@ install-dev:
bootstrap:
$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
$(UV_RUN) python scripts/prisma_generate_if_needed.py
cd ui/litellm-dashboard && npm install --no-audit --no-fund
cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund
@main_root=$$(git worktree list --porcelain | head -1 | sed 's/^worktree //'); \
if [ "$$main_root" != "$$(git rev-parse --show-toplevel)" ] && [ -f "$$main_root/.env" ] && [ ! -f .env ]; then \
cp "$$main_root/.env" .env && echo "bootstrap: copied .env from $$main_root"; \
@ -176,10 +176,8 @@ lint-ruff-FULL-dev: install-dev
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
else echo "No changed .py files to check."; fi
lint-basedpyright lint-basedpyright-budget-update: export NODE_OPTIONS := --max-old-space-size=12288
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
$(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e
@ -192,7 +190,7 @@ lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev lint-fetch-base
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
$(UV_RUN) python scripts/type_check_gate.py --update
lint-format: format-check

1
backend/routes/allowlist.py
View file

@ -44,6 +44,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/router/",
"/router_settings",
"/adaptive_router/",
"/auto_router/",
"/fallback",
"/fallbacks",
"/cache_settings",

34
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 29813
"limit": 29806
},
"reportArgumentType": {
"limit": 2645
@ -18,28 +18,28 @@
"limit": 40
},
"reportDeprecated": {
"limit": 325
"limit": 215
},
"reportDuplicateImport": {
"limit": 24
"limit": 19
},
"reportExplicitAny": {
"limit": 9473
"limit": 9469
},
"reportFunctionMemberAccess": {
"limit": 11
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 157
},
"reportIncompatibleMethodOverride": {
"limit": 77
"limit": 56
},
"reportIncompatibleVariableOverride": {
"limit": 12
"limit": 8
},
"reportInconsistentOverload": {
"limit": 18
"limit": 12
},
"reportIndexIssue": {
"limit": 35
@ -48,7 +48,7 @@
"limit": 35
},
"reportInvalidTypeVarUse": {
"limit": 5
"limit": 2
},
"reportMatchNotExhaustive": {
"limit": 0
@ -60,7 +60,7 @@
"limit": 15849
},
"reportMissingTypeStubs": {
"limit": 41
"limit": 40
},
"reportOperatorIssue": {
"limit": 0
@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 2436
"limit": 1825
},
"reportRedeclaration": {
"limit": 8
@ -99,19 +99,19 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 45269
"limit": 45262
},
"reportUnknownLambdaType": {
"limit": 113
},
"reportUnknownMemberType": {
"limit": 40452
"limit": 40447
},
"reportUnknownParameterType": {
"limit": 20309
},
"reportUnknownVariableType": {
"limit": 31978
"limit": 31880
},
"reportUnnecessaryCast": {
"limit": 124
@ -126,7 +126,7 @@
"limit": 866
},
"reportUntypedBaseClass": {
"limit": 165
"limit": 72
},
"reportUntypedFunctionDecorator": {
"limit": 33
@ -138,9 +138,9 @@
"limit": 139
},
"reportUnusedImport": {
"limit": 588
"limit": 556
},
"reportUnusedVariable": {
"limit": 147
"limit": 146
}
}

0
enterprise/litellm_enterprise/enterprise_callbacks/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/integrations/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/litellm_core_utils/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/proxy/hooks/__init__.py Normal file
View file

6
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -382,7 +382,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids]
return [
OpenAIFileObject.model_validate(file_object.file_object)
for file_object in file_ids
if file_object.file_object is not None
]
async def check_managed_file_id_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth

4
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -831,7 +831,7 @@ async def project_info(
)
# Check if user has access to this project (admin or team member)
is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
is_admin = user_api_key_has_admin_view(user_api_key_dict)
is_team_member = False
if project.team_id and user_api_key_dict.user_id:
@ -886,7 +886,7 @@ async def list_projects(
)
# If proxy admin, get all projects
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
if user_api_key_has_admin_view(user_api_key_dict):
projects: Sequence[
prisma_models.LiteLLM_ProjectTable
] = await prisma_client.db.litellm_projecttable.find_many(

0
enterprise/litellm_enterprise/proxy/vector_stores/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/py.typed Normal file
View file

0
enterprise/litellm_enterprise/types/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/types/enterprise_callbacks/__init__.py Normal file
View file

0
enterprise/litellm_enterprise/types/proxy/__init__.py Normal file
View file

177
litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py Normal file
View file

@ -0,0 +1,177 @@
"""Prepare the Node toolchain the Prisma CLI needs, separately from migrations.
The Prisma CLI is a Node program. The first invocation inside a fresh
container installs a private Node runtime and npm-installs the CLI itself,
which can take minutes on a cold or slow machine. Sharing one timeout between
that one-time bootstrap and the migration commands makes a slow bootstrap
indistinguishable from a slow migration, so the bootstrap gets killed long
before it can finish.
A killed bootstrap does not correct itself. The installer leaves its cache
directory behind, and Prisma decides whether to install by testing that
directory for existence alone, so every later attempt skips the install and
then fails on a Node binary that was never written. Deleting a cache directory
that exists without a Node binary is what turns a killed bootstrap back into a
recoverable one.
Both budgets are overridable so an operator can widen them without a release:
``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install and
``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every individual Prisma command.
"""
import math
import os
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
from litellm_proxy_extras._logging import logger
try:
from prisma import config as prisma_config
except ImportError:
prisma_config = None
PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
BOOTSTRAP_ARG = "--version"
@dataclass(frozen=True)
class ToolchainBootstrap:
"""Outcome of preparing the Prisma toolchain."""
healed_incomplete_cache: bool
ready: bool
def _timeout_from_env(env_var: str, default: float) -> float:
raw = os.getenv(env_var)
if raw is None:
return default
try:
seconds = float(raw)
except ValueError:
logger.warning(
"%s=%r is not a number, falling back to %ss", env_var, raw, default
)
return default
if not math.isfinite(seconds) or seconds <= 0:
logger.warning(
"%s=%r is not a finite positive number, falling back to %ss",
env_var,
raw,
default,
)
return default
return seconds
def prisma_command_timeout() -> float:
"""Seconds any single Prisma command may run for."""
return _timeout_from_env(
PRISMA_COMMAND_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_COMMAND_TIMEOUT
)
def prisma_bootstrap_timeout() -> float:
"""Seconds the one-time Node toolchain install may run for."""
return _timeout_from_env(
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT
)
def nodeenv_cache_dir() -> Optional[Path]:
"""Where Prisma installs its private Node runtime, or None if unknowable."""
override = os.getenv(NODEENV_CACHE_DIR_ENV_VAR)
if override:
return Path(override).absolute()
if prisma_config is not None:
try:
return Path(prisma_config.nodeenv_cache_dir).absolute()
except (OSError, ValueError) as e:
logger.warning("Could not read the Prisma nodeenv cache dir: %s", e)
try:
return Path.home() / ".cache" / "prisma-python" / "nodeenv"
except RuntimeError:
logger.warning(
"No resolvable home directory, cannot locate the Prisma nodeenv cache"
)
return None
def node_binary_path(cache_dir: Path) -> Path:
"""Path the Node binary occupies once the toolchain is fully installed."""
if os.name == "nt":
return cache_dir / "Scripts" / "node.exe"
return cache_dir / "bin" / "node"
def heal_incomplete_nodeenv_cache() -> bool:
"""Delete a nodeenv cache directory left without a Node binary.
Returns True when a half-installed toolchain was removed, so the next
Prisma invocation reinstalls it instead of failing on a missing binary.
"""
cache_dir = nodeenv_cache_dir()
if cache_dir is None or not cache_dir.is_dir():
return False
if node_binary_path(cache_dir).exists():
return False
logger.warning(
"Node toolchain at %s has no %s, so a previous install was interrupted. "
"Removing it so it can be reinstalled.",
cache_dir,
node_binary_path(cache_dir).name,
)
try:
shutil.rmtree(cache_dir)
except OSError as e:
logger.warning("Could not remove %s: %s", cache_dir, e)
return False
return True
def ensure_prisma_toolchain(
prisma_command: str, prisma_env: dict[str, str]
) -> ToolchainBootstrap:
"""Install whatever the Prisma CLI needs to run, under its own timeout.
Never raises. A toolchain that cannot be prepared is reported so the
caller can go on and let the real Prisma command produce the real error.
"""
healed = heal_incomplete_nodeenv_cache()
timeout = prisma_bootstrap_timeout()
logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout)
try:
subprocess.run(
[prisma_command, BOOTSTRAP_ARG],
timeout=timeout,
check=True,
capture_output=True,
text=True,
env=prisma_env,
)
except subprocess.TimeoutExpired:
logger.warning(
"Preparing the Prisma CLI toolchain timed out after %ss. Raise %s "
"if this machine needs longer to install it.",
timeout,
PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR,
)
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=False)
except subprocess.CalledProcessError as e:
logger.warning("Preparing the Prisma CLI toolchain failed: %s", e.stderr)
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=False)
except OSError as e:
logger.warning("Could not run the Prisma CLI: %s", e)
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=False)
logger.info("Prisma CLI toolchain ready")
return ToolchainBootstrap(healed_incomplete_cache=healed, ready=True)

0
litellm-proxy-extras/litellm_proxy_extras/py.typed Normal file
View file

3
litellm-proxy-extras/litellm_proxy_extras/replica_identity.py
View file

@ -16,6 +16,7 @@ import tempfile
from pathlib import Path
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout
REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL"
@ -75,7 +76,7 @@ def apply_replica_identity_full(
"--schema",
schema_path,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,

33
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -14,6 +14,10 @@ from litellm_proxy_extras.replica_identity import (
REPLICA_IDENTITY_FULL_ENV_VAR,
apply_replica_identity_full,
)
from litellm_proxy_extras.prisma_toolchain import (
ensure_prisma_toolchain,
prisma_command_timeout,
)
def str_to_bool(value: Optional[str]) -> bool:
@ -142,7 +146,7 @@ class ProxyExtrasDBManager:
],
stdout=open(migration_file, "w"),
check=True,
timeout=30,
timeout=prisma_command_timeout(),
env=prisma_env,
)
@ -157,7 +161,7 @@ class ProxyExtrasDBManager:
"0_init",
],
check=True,
timeout=30,
timeout=prisma_command_timeout(),
env=prisma_env,
)
@ -193,7 +197,7 @@ class ProxyExtrasDBManager:
"--rolled-back",
migration_name,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
env=prisma_env,
@ -205,7 +209,7 @@ class ProxyExtrasDBManager:
prisma_env = _get_prisma_env()
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
env=prisma_env,
@ -303,7 +307,7 @@ class ProxyExtrasDBManager:
"--script",
],
check=True,
timeout=60,
timeout=prisma_command_timeout(),
stdout=f,
env=_get_prisma_env(),
)
@ -335,7 +339,7 @@ class ProxyExtrasDBManager:
"--schema",
schema_path,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -364,7 +368,7 @@ class ProxyExtrasDBManager:
"--schema",
schema_path,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -393,7 +397,7 @@ class ProxyExtrasDBManager:
"--applied",
migration_name,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -530,7 +534,7 @@ class ProxyExtrasDBManager:
try:
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
env=_get_prisma_env(),
)
@ -555,7 +559,7 @@ class ProxyExtrasDBManager:
try:
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -731,6 +735,9 @@ class ProxyExtrasDBManager:
Returns:
bool: True if setup was successful, False otherwise
"""
ensure_prisma_toolchain(
prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env()
)
migrated = ProxyExtrasDBManager._run_migrations(
use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
)
@ -757,7 +764,7 @@ class ProxyExtrasDBManager:
# Set migrations directory for Prisma
result = subprocess.run(
[_get_prisma_command(), "migrate", "deploy"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -840,7 +847,7 @@ class ProxyExtrasDBManager:
"--rolled-back",
failed_migration,
],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
capture_output=True,
text=True,
@ -968,7 +975,7 @@ class ProxyExtrasDBManager:
# Use prisma db push with increased timeout
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=60,
timeout=prisma_command_timeout(),
check=True,
)
return True

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.82"
version = "0.4.83"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.82"
version = "0.4.83"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

44
litellm/__init__.py
View file

@ -1269,8 +1269,8 @@ from .llms.xai.common_utils import XAIModelInfo
from litellm.types.utils import LlmProviders
## Lazy loading this is not straightforward, will leave it here for now.
from .main import * # type: ignore
from .compression import compress # type: ignore[no-redef]
from .main import *
from .compression import compress
# Skills API
from .skills.main import (
@ -1341,7 +1341,7 @@ from .assistants.main import *
from .batches.main import *
from .images.main import *
from .videos.main import *
from .batch_completion.main import * # type: ignore
from .batch_completion.main import *
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
@ -2054,7 +2054,7 @@ if TYPE_CHECKING:
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType] # type: ignore[no-redef]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
validate_environment: Callable[..., dict]
check_valid_key: Callable[..., bool]
@ -2150,9 +2150,9 @@ def __getattr__(name: str) -> Any:
# Lazy load encoding from main.py to avoid heavy tiktoken import
if name == "encoding":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "encoding" not in _globals:
from .main import encoding as _encoding
@ -2162,9 +2162,9 @@ def __getattr__(name: str) -> Any:
# Lazy load bedrock_tool_name_mappings instance
if name == "bedrock_tool_name_mappings":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "bedrock_tool_name_mappings" not in _globals:
from .llms.bedrock.chat.invoke_handler import (
@ -2176,9 +2176,9 @@ def __getattr__(name: str) -> Any:
# Lazy load AzureOpenAIError exception class
if name == "AzureOpenAIError":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "AzureOpenAIError" not in _globals:
from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
@ -2188,9 +2188,9 @@ def __getattr__(name: str) -> Any:
# Lazy load openaiOSeriesConfig instance
if name == "openaiOSeriesConfig":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
if "openaiOSeriesConfig" not in _globals:
# Import the config class and instantiate it
config_class = __getattr__("OpenAIOSeriesConfig")
@ -2206,9 +2206,9 @@ def __getattr__(name: str) -> Any:
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
}
if name in _config_instances:
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
if name not in _globals:
# Import the config class and instantiate it
config_class = __getattr__(_config_instances[name])
@ -2221,9 +2221,9 @@ def __getattr__(name: str) -> Any:
# Lazy load provider_list
if name == "provider_list":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "provider_list" not in _globals:
# LlmProviders is eagerly imported above, so we can import it directly
@ -2234,9 +2234,9 @@ def __getattr__(name: str) -> Any:
# Lazy load priority_reservation_settings instance
if name == "priority_reservation_settings":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "priority_reservation_settings" not in _globals:
# Import the class and instantiate it
@ -2246,9 +2246,9 @@ def __getattr__(name: str) -> Any:
# Lazy load logging_callback_manager instance
if name == "logging_callback_manager":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "logging_callback_manager" not in _globals:
# Import the class and instantiate it
@ -2258,9 +2258,9 @@ def __getattr__(name: str) -> Any:
# Lazy load _service_logger module
if name == "_service_logger":
from ._lazy_imports import _get_litellm_globals
from ._lazy_imports import get_litellm_globals
_globals = _get_litellm_globals()
_globals = get_litellm_globals()
# Check if already cached
if "_service_logger" not in _globals:
# Import the module lazily

10
litellm/_lazy_imports.py
View file

@ -54,7 +54,7 @@ from ._lazy_imports_registry import (
)
def _get_litellm_globals() -> dict:
def get_litellm_globals() -> dict:
"""
Get the globals dictionary of the litellm module.
@ -233,7 +233,7 @@ def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], cate
raise AttributeError(f"{category} lazy import: unknown attribute {name!r}")
# Step 2: Get the cache (where we store imported things)
_globals: Final = _get_litellm_globals()
_globals: Final = get_litellm_globals()
# Step 3: If we've already imported it, just return the cached version
if name in _globals:
@ -332,7 +332,7 @@ def _lazy_import_utils_module(name: str) -> Any:
Handler for utils module lazy imports.
This uses a custom implementation because utils module needs to use
_get_utils_globals() instead of _get_litellm_globals() for caching.
_get_utils_globals() instead of get_litellm_globals() for caching.
"""
# Check if this attribute exists in our map
if name not in _UTILS_MODULE_IMPORT_MAP:
@ -379,7 +379,7 @@ def _lazy_import_llm_client_cache(name: str) -> Any:
- "in_memory_llm_clients_cache" is a singleton instance of that class
So we need custom logic to handle both cases.
"""
_globals: Final = _get_litellm_globals()
_globals: Final = get_litellm_globals()
# If already cached, return it
if name in _globals:
@ -412,7 +412,7 @@ def _lazy_import_http_handlers(name: str) -> Any:
- They need configuration (timeout, etc.) from the module globals
- They use factory functions instead of direct instantiation
"""
_globals: Final = _get_litellm_globals()
_globals: Final = get_litellm_globals()
if name == "module_level_aclient":
# Create an async HTTP client using the factory function

30
litellm/_lazy_imports_registry.py
View file

@ -1461,32 +1461,30 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
# Export all name tuples and import maps for use in _lazy_imports.py
__all__ = [
# Name tuples
"COST_CALCULATOR_NAMES",
"LITELLM_LOGGING_NAMES",
"UTILS_NAMES",
"TOKEN_COUNTER_NAMES",
"LLM_CLIENT_CACHE_NAMES",
"BEDROCK_TYPES_NAMES",
"TYPES_UTILS_NAMES",
"CACHING_NAMES",
"HTTP_HANDLER_NAMES",
"COST_CALCULATOR_NAMES",
"DOTPROMPT_NAMES",
"HTTP_HANDLER_NAMES",
"LITELLM_LOGGING_NAMES",
"LLM_CLIENT_CACHE_NAMES",
"LLM_CONFIG_NAMES",
"TYPES_NAMES",
"LLM_PROVIDER_LOGIC_NAMES",
"TOKEN_COUNTER_NAMES",
"TYPES_NAMES",
"TYPES_UTILS_NAMES",
"UTILS_MODULE_NAMES",
# Import maps
"_UTILS_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
"_TYPES_UTILS_IMPORT_MAP",
"_TOKEN_COUNTER_IMPORT_MAP",
"UTILS_NAMES",
"_BEDROCK_TYPES_IMPORT_MAP",
"_CACHING_IMPORT_MAP",
"_LITELLM_LOGGING_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
"_DOTPROMPT_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_LITELLM_LOGGING_IMPORT_MAP",
"_LLM_CONFIGS_IMPORT_MAP",
"_LLM_PROVIDER_LOGIC_IMPORT_MAP",
"_TOKEN_COUNTER_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_TYPES_UTILS_IMPORT_MAP",
"_UTILS_IMPORT_MAP",
"_UTILS_MODULE_IMPORT_MAP",
]

26
litellm/_redis.py
View file

@ -15,8 +15,8 @@ import os
from collections.abc import Callable
from typing import Final
import redis # type: ignore
import redis.asyncio as async_redis # type: ignore
import redis
import redis.asyncio as async_redis
from litellm import get_secret, get_secret_str
from litellm._redis_credential_provider import (
@ -153,7 +153,7 @@ def _redis_kwargs_from_environment():
return_dict: Final = {}
for k, v in mapping.items():
value = get_secret(k, default_value=None) # type: ignore
value = get_secret(k, default_value=None)
if value is not None:
return_dict[v] = value
return return_dict
@ -317,7 +317,7 @@ def create_azure_ad_redis_connect_func(
# AzureADCredentialProvider for refresh-aware token retrieval. The raw
# client_id/tenant_id/secret are intentionally NOT exposed here — the
# credential closure already holds them.
ad_connect._azure_credential = credential # type: ignore[attr-defined]
ad_connect._azure_credential = credential
return ad_connect
@ -351,7 +351,7 @@ def _get_redis_client_logic(**env_overrides):
for k, v in env_overrides.items():
if isinstance(v, str) and v.startswith("os.environ/"):
v = v.replace("os.environ/", "")
value = get_secret(v) # type: ignore
value = get_secret(v)
env_overrides[k] = value
environment_kwargs: Final = _redis_kwargs_from_environment()
@ -370,7 +370,7 @@ def _get_redis_client_logic(**env_overrides):
**env_overrides,
}
_startup_nodes: Final[str | list | None] = redis_kwargs.get("startup_nodes", None) or get_secret( # type: ignore
_startup_nodes: Final[str | list | None] = redis_kwargs.get("startup_nodes", None) or get_secret(
"REDIS_CLUSTER_NODES"
)
@ -381,7 +381,7 @@ def _get_redis_client_logic(**env_overrides):
elif _startup_nodes is None:
redis_kwargs.pop("startup_nodes", None)
_sentinel_nodes: Final[str | list | None] = redis_kwargs.get("sentinel_nodes", None) or get_secret( # type: ignore
_sentinel_nodes: Final[str | list | None] = redis_kwargs.get("sentinel_nodes", None) or get_secret(
"REDIS_SENTINEL_NODES"
)
@ -395,7 +395,7 @@ def _get_redis_client_logic(**env_overrides):
if _sentinel_password is not None:
redis_kwargs["sentinel_password"] = _sentinel_password
_service_name: Final[str | None] = redis_kwargs.get("service_name", None) or get_secret( # type: ignore
_service_name: Final[str | None] = redis_kwargs.get("service_name", None) or get_secret(
"REDIS_SERVICE_NAME"
)
@ -412,7 +412,7 @@ def _get_redis_client_logic(**env_overrides):
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account # type: ignore[attr-defined]
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
@ -449,7 +449,7 @@ def _get_redis_client_logic(**env_overrides):
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True # type: ignore[attr-defined]
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
@ -481,7 +481,7 @@ def _get_redis_client_logic(**env_overrides):
def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
_redis_cluster_nodes_in_env: Final[str | None] = get_secret("REDIS_CLUSTER_NODES") # type: ignore
_redis_cluster_nodes_in_env: Final[str | None] = get_secret("REDIS_CLUSTER_NODES")
if _redis_cluster_nodes_in_env is not None:
try:
redis_kwargs["startup_nodes"] = json.loads(_redis_cluster_nodes_in_env)
@ -505,7 +505,7 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
new_startup_nodes.append(ClusterNode(**item))
cluster_kwargs.pop("startup_nodes", None)
return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs) # type: ignore
return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs)
def _get_redis_sentinel_connection_kwargs(redis_kwargs: dict) -> dict:
@ -638,7 +638,7 @@ def get_redis_async_client(
# Create async RedisCluster with IAM token as password if available
cluster_client: Final = async_redis.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs, # type: ignore
**cluster_kwargs,
)
return cluster_client

2
litellm/_redis_credential_provider.py
View file

@ -3,7 +3,7 @@ import threading
import time
from typing import Any, Final
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
from redis.credentials import CredentialProvider
# Azure AD scope for Redis Cache for Azure.
AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default"

4
litellm/_service_logger.py
View file

@ -1,6 +1,6 @@
import asyncio
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm._logging import verbose_logger
@ -16,7 +16,7 @@ if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
Span = Union[_Span, Any]
Span = _Span | Any
OTELClass = OpenTelemetry
else:
Span = Any

2
litellm/_uuid.py
View file

@ -4,7 +4,7 @@ Internal unified UUID helper.
Always uses fastuuid for performance.
"""
import fastuuid as _uuid # type: ignore
import fastuuid as _uuid
# Expose a module-like alias so callers can use: uuid.uuid4()
uuid = _uuid

22
litellm/a2a_protocol/__init__.py
View file

@ -55,19 +55,15 @@ from litellm.a2a_protocol.main import (
from litellm.types.agents import LiteLLMSendMessageResponse
__all__ = [
# Client
"A2AClient",
# Functions
"asend_message",
"send_message",
"asend_message_streaming",
"aget_agent_card",
"create_a2a_client",
# Response types
"LiteLLMSendMessageResponse",
# Exceptions
"A2AError",
"A2AConnectionError",
"A2AAgentCardError",
"A2AClient",
"A2AConnectionError",
"A2AError",
"A2ALocalhostURLError",
"LiteLLMSendMessageResponse",
"aget_agent_card",
"asend_message",
"asend_message_streaming",
"create_a2a_client",
"send_message",
]

6
litellm/a2a_protocol/card_resolver.py
View file

@ -18,8 +18,8 @@ AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent-card.json"
PREV_AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent.json"
try:
from a2a.client import A2ACardResolver as _A2ACardResolver # type: ignore[no-redef]
from a2a.utils.constants import ( # type: ignore[no-redef]
from a2a.client import A2ACardResolver as _A2ACardResolver
from a2a.utils.constants import (
AGENT_CARD_WELL_KNOWN_PATH,
PREV_AGENT_CARD_WELL_KNOWN_PATH,
)
@ -102,7 +102,7 @@ def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
return agent_card
class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
class LiteLLMA2ACardResolver(_A2ACardResolver):
"""
Custom A2A card resolver that supports multiple well-known paths.

10
litellm/a2a_protocol/exception_mapping_utils.py
View file

@ -29,9 +29,9 @@ try:
A2A_SDK_AVAILABLE = True
except ImportError:
A2A_SDK_AVAILABLE = False
Client = None # type: ignore[misc, assignment]
ClientConfig = None # type: ignore[misc, assignment]
create_client = None # type: ignore[misc, assignment]
Client = None
ClientConfig = None
create_client = None
class A2AExceptionCheckers:
@ -219,6 +219,6 @@ async def handle_a2a_localhost_retry(
streaming=is_streaming,
),
)
new_client._litellm_httpx_client = httpx_client # type: ignore[attr-defined]
new_client._litellm_agent_card = agent_card # type: ignore[attr-defined]
new_client._litellm_httpx_client = httpx_client
new_client._litellm_agent_card = agent_card
return new_client

2
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -271,7 +271,7 @@ class A2ACompletionBridgeHandler:
# 3. Accumulate content and emit artifact update
accumulated_text = ""
chunk_count = 0
async for chunk in response: # type: ignore[union-attr]
async for chunk in response:
chunk_count += 1
# Extract delta content

10
litellm/a2a_protocol/main.py
View file

@ -59,9 +59,9 @@ try:
A2A_SDK_AVAILABLE = True
except ImportError:
Client = None # type: ignore[misc, assignment]
ClientConfig = None # type: ignore[misc, assignment]
create_client = None # type: ignore[misc, assignment]
Client = None
ClientConfig = None
create_client = None
# Import our custom card resolver that supports multiple well-known paths
from litellm.a2a_protocol.card_resolver import (
@ -788,10 +788,10 @@ async def create_a2a_client(
# Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
# the configured httpx client (with this agent's trace-id/auth headers) without
# excavating a2a-sdk private internals.
a2a_client._litellm_httpx_client = httpx_client # type: ignore[attr-defined]
a2a_client._litellm_httpx_client = httpx_client
agent_card: Final = getattr(a2a_client, "_card", None)
if agent_card is not None:
a2a_client._litellm_agent_card = agent_card # type: ignore[attr-defined]
a2a_client._litellm_agent_card = agent_card
verbose_logger.info("A2A client created for %s", base_url)

2
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -153,7 +153,7 @@ class AnthropicExceptionMapping:
# Optionally add request_id if provided and not present
if request_id and "request_id" not in parsed:
parsed["request_id"] = request_id
return parsed # type: ignore
return parsed
# Extract message - use parsed dict if available, otherwise raw string
if parsed is not None:

170
litellm/assistants/main.py
View file

@ -51,9 +51,7 @@ async def aget_assistants(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -61,7 +59,7 @@ async def aget_assistants(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -98,7 +96,7 @@ def get_assistants(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -132,12 +130,12 @@ def get_assistants(
max_retries=optional_params.max_retries,
organization=organization,
client=client,
aget_assistants=aget_assistants, # type: ignore
) # type: ignore
aget_assistants=aget_assistants,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -145,14 +143,14 @@ def get_assistants(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.get_assistants(
api_base=api_base,
@ -162,7 +160,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
client=client,
aget_assistants=aget_assistants, # type: ignore
aget_assistants=aget_assistants,
litellm_params=litellm_params_dict,
)
else:
@ -173,7 +171,7 @@ def get_assistants(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
@ -185,7 +183,7 @@ def get_assistants(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
@ -210,9 +208,7 @@ async def acreate_assistants(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model=model, custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -220,7 +216,7 @@ async def acreate_assistants(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model=model,
@ -267,7 +263,7 @@ def create_assistants(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -318,12 +314,12 @@ def create_assistants(
organization=organization,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants, # type: ignore
) # type: ignore
async_create_assistants=async_create_assistants,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -331,14 +327,14 @@ def create_assistants(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -363,7 +359,7 @@ def create_assistants(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
if response is None:
@ -392,9 +388,7 @@ async def adelete_assistant(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -402,7 +396,7 @@ async def adelete_assistant(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -442,7 +436,7 @@ def delete_assistant(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -472,9 +466,9 @@ def delete_assistant(
async_delete_assistants=async_delete_assistants,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -482,14 +476,14 @@ def delete_assistant(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -541,9 +535,7 @@ async def acreate_thread(custom_llm_provider: Literal["openai", "azure"], **kwar
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -551,7 +543,7 @@ async def acreate_thread(custom_llm_provider: Literal["openai", "azure"], **kwar
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -608,7 +600,7 @@ def create_thread(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -649,7 +641,7 @@ def create_thread(
acreate_thread=acreate_thread,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_key = (
optional_params.api_key
@ -657,16 +649,16 @@ def create_thread(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -692,10 +684,10 @@ def create_thread(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
async def aget_thread(
@ -715,9 +707,7 @@ async def aget_thread(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -725,7 +715,7 @@ async def aget_thread(
response = await init_response
else:
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -758,7 +748,7 @@ def get_thread(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
api_base: str | None = None
@ -797,9 +787,9 @@ def get_thread(
aget_thread=aget_thread,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -807,14 +797,14 @@ def get_thread(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
if isinstance(client, OpenAI):
client = None # only pass client if it's AzureOpenAI
@ -839,10 +829,10 @@ def get_thread(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
### MESSAGES ###
@ -879,9 +869,7 @@ async def a_add_message(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -890,7 +878,7 @@ async def a_add_message(
else:
# Call the synchronous function using run_in_executor
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -937,7 +925,7 @@ def add_message(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
api_key: str | None = None
@ -976,9 +964,9 @@ def add_message(
a_add_message=a_add_message,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -986,14 +974,14 @@ def add_message(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.add_message(
thread_id=thread_id,
@ -1016,11 +1004,11 @@ def add_message(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
async def aget_messages(
@ -1046,9 +1034,7 @@ async def aget_messages(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -1057,7 +1043,7 @@ async def aget_messages(
else:
# Call the synchronous function using run_in_executor
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -1090,7 +1076,7 @@ def get_messages(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -1129,9 +1115,9 @@ def get_messages(
aget_messages=aget_messages,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -1139,14 +1125,14 @@ def get_messages(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token: str | None = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.get_messages(
thread_id=thread_id,
@ -1168,11 +1154,11 @@ def get_messages(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response
### RUNS ###
@ -1182,7 +1168,7 @@ def arun_thread_stream(
**kwargs,
) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]:
kwargs["arun_thread"] = True
return run_thread(stream=True, event_handler=event_handler, **kwargs) # type: ignore
return run_thread(stream=True, event_handler=event_handler, **kwargs)
async def arun_thread(
@ -1222,9 +1208,7 @@ async def arun_thread(
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider( # type: ignore
model="", custom_llm_provider=custom_llm_provider
) # type: ignore
_, custom_llm_provider, _, _ = get_llm_provider(model="", custom_llm_provider=custom_llm_provider)
# Await normally
init_response: Final = await loop.run_in_executor(None, func_with_context)
@ -1233,7 +1217,7 @@ async def arun_thread(
else:
# Call the synchronous function using run_in_executor
response = init_response
return response # type: ignore
return response
except Exception as e:
raise exception_type(
model="",
@ -1249,7 +1233,7 @@ def run_thread_stream(
event_handler: AssistantEventHandler | None = None,
**kwargs,
) -> AssistantStreamManager[AssistantEventHandler]:
return run_thread(stream=True, event_handler=event_handler, **kwargs) # type: ignore
return run_thread(stream=True, event_handler=event_handler, **kwargs)
def run_thread(
@ -1283,7 +1267,7 @@ def run_thread(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -1329,9 +1313,9 @@ def run_thread(
event_handler=event_handler,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -1339,14 +1323,14 @@ def run_thread(
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
) # type: ignore
)
extra_body: Final = optional_params.get("extra_body", {})
azure_ad_token = None
if extra_body is not None:
azure_ad_token = extra_body.pop("azure_ad_token", None)
else:
azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore
azure_ad_token = get_secret("AZURE_AD_TOKEN")
response = azure_assistants_api.run_thread(
thread_id=thread_id,
@ -1366,7 +1350,7 @@ def run_thread(
client=client,
arun_thread=arun_thread,
litellm_params=litellm_params_dict,
) # type: ignore
)
else:
raise litellm.exceptions.BadRequestError(
message=f"LiteLLM doesn't support {custom_llm_provider} for 'run_thread'. Only 'openai' is supported.",
@ -1375,7 +1359,7 @@ def run_thread(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response # type: ignore
return response

8
litellm/assistants/utils.py
View file

@ -8,8 +8,8 @@ from ..types.llms.openai import *
def get_optional_params_add_message(
role: str | None,
content: str | List[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
attachments: List[Attachment] | None,
content: str | list[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
attachments: list[Attachment] | None,
metadata: dict | None,
custom_llm_provider: str,
**kwargs,
@ -57,7 +57,7 @@ def get_optional_params_add_message(
optional_params = litellm.AzureOpenAIAssistantsAPIConfig().map_openai_params_create_message_params(
non_default_params=non_default_params, optional_params=optional_params
)
for k in passed_params.keys():
for k in passed_params:
if k not in default_params:
optional_params[k] = passed_params[k]
return optional_params
@ -128,7 +128,7 @@ def get_optional_params_image_gen(
if n is not None:
optional_params["sampleCount"] = int(n)
for k in passed_params.keys():
for k in passed_params:
if k not in default_params:
optional_params[k] = passed_params[k]
return optional_params

6
litellm/batches/batch_utils.py
View file

@ -274,7 +274,7 @@ async def _fetch_batch_output_file_content(
credentials: Final = _extract_file_access_credentials(litellm_params)
file_content_kwargs.update(credentials)
_file_content: Final = await afile_content(**file_content_kwargs) # type: ignore[reportArgumentType]
_file_content: Final = await afile_content(**file_content_kwargs)
return _file_content.content
@ -432,7 +432,11 @@ def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_prov
usage_object=response_body.get("usage", None) or {},
reasoning_content=None,
)
from litellm.responses.utils import ResponseAPILoggingUtils
_usage_dict: Final = response_body.get("usage", None) or {}
if ResponseAPILoggingUtils._is_response_api_usage(_usage_dict):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_usage_dict)
usage: Final[Usage] = Usage(**_usage_dict)
return usage

32
litellm/batches/main.py
View file

@ -104,7 +104,7 @@ def _resolve_timeout(
@client
async def acreate_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
input_file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
metadata: dict[str, str] | None = None,
@ -154,7 +154,7 @@ async def acreate_batch(
@client
def create_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
input_file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
metadata: dict[str, str] | None = None,
@ -287,7 +287,7 @@ def create_batch(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.create_batch(
_is_async=_is_async,
@ -327,7 +327,7 @@ def create_batch(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_batch", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -370,7 +370,7 @@ async def aretrieve_batch(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -436,7 +436,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.retrieve_batch(
_is_async=_is_async,
@ -498,7 +498,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -545,7 +545,7 @@ def retrieve_batch(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -677,7 +677,7 @@ async def alist_batches(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -723,7 +723,7 @@ def list_batches(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -755,7 +755,7 @@ def list_batches(
max_retries=optional_params.max_retries,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
@ -770,7 +770,7 @@ def list_batches(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.list_batches(
_is_async=_is_async,
@ -813,7 +813,7 @@ def list_batches(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -909,7 +909,7 @@ def cancel_batch(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -959,7 +959,7 @@ def cancel_batch(
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
get_secret_str("AZURE_AD_TOKEN")
response = azure_batches_instance.cancel_batch(
_is_async=_is_async,
@ -999,7 +999,7 @@ def cancel_batch(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="cancel_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="cancel_batch", url="https://github.com/BerriAI/litellm"),
),
)
return response

4
litellm/caching/base_cache.py
View file

@ -9,12 +9,12 @@ Has 4 methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any

6
litellm/caching/caching.py
View file

@ -534,11 +534,9 @@ class Cache:
if isinstance(cached_response, dict):
pass
else:
cached_response = json.loads(
cached_response # type: ignore
) # Convert string to dictionary
cached_response = json.loads(cached_response) # Convert string to dictionary
except Exception:
cached_response = ast.literal_eval(cached_response) # type: ignore
cached_response = ast.literal_eval(cached_response)
return cached_response
return cached_result

4
litellm/caching/caching_handler.py
View file

@ -242,7 +242,7 @@ class LLMCachingHandler:
or litellm.cache.get_cache_key(**self.request_kwargs)
)
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
cached_result._hidden_params["cache_key"] = cache_key
return CachingHandlerResponse(cached_result=cached_result)
elif (
call_type == CallTypes.aembedding.value
@ -356,7 +356,7 @@ class LLMCachingHandler:
or litellm.cache.get_cache_key(**self.request_kwargs)
)
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
cached_result._hidden_params["cache_key"] = cache_key
return CachingHandlerResponse(cached_result=cached_result)
return CachingHandlerResponse(cached_result=cached_result)

6
litellm/caching/disk_cache.py
View file

@ -1,12 +1,12 @@
import json
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any
@ -44,7 +44,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response) # type: ignore
cached_response = json.loads(original_cached_response)
except Exception:
cached_response = original_cached_response
return cached_response

4
litellm/caching/dual_cache.py
View file

@ -13,7 +13,7 @@ import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from litellm.types.caching import RedisPipelineIncrementOperation
@ -29,7 +29,7 @@ from .redis_cache import RedisCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any

5
litellm/caching/evicted_client_closer.py
View file

@ -34,6 +34,7 @@ alive anything the collector would have reclaimed first.
"""
import asyncio
import contextlib
import inspect
import threading
import time
@ -146,10 +147,8 @@ def _has_connection_in_flight(client: object) -> bool:
async def _close_quietly(closing: Awaitable[object]) -> None:
try:
with contextlib.suppress(Exception):
await closing
except Exception: # noqa: BLE001 - a discarded client's close must never surface to callers
pass
class EvictedClientCloser:

2
litellm/caching/llm_caching_handler.py
View file

@ -29,7 +29,7 @@ class LLMClientCache(InMemoryCache):
default_ttl: int | None = 600,
max_size_per_item: int | None = 1024,
evicted_client_closer: EvictedClientCloser | None = None,
):
) -> None:
super().__init__(
max_size_in_memory=max_size_in_memory,
default_ttl=default_ttl,

50
litellm/caching/redis_cache.py
View file

@ -18,7 +18,7 @@ import time
from collections.abc import Awaitable, Callable, Sequence
from contextvars import ContextVar
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final, TypeVar, Union, cast
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
@ -49,7 +49,7 @@ if TYPE_CHECKING:
cluster_pipeline = ClusterPipeline
async_redis_client = Redis
async_redis_cluster_client = RedisCluster
Span = Union[_Span, Any]
Span = _Span | Any
else:
pipeline = Any
cluster_pipeline = Any
@ -242,7 +242,7 @@ async def _run_under_circuit_breaker(
return result
def _redis_circuit_breaker_guard(method): # type: ignore
def _redis_circuit_breaker_guard(method):
"""
Decorator for RedisCache async methods.
Checks the circuit breaker before each call; records success/failure after.
@ -256,7 +256,7 @@ def _redis_circuit_breaker_guard(method): # type: ignore
"""
@functools.wraps(method)
async def wrapper(self, *args, **kwargs): # type: ignore
async def wrapper(self, *args, **kwargs):
return await _run_under_circuit_breaker(
self._circuit_breaker, method.__name__, lambda: method(self, *args, **kwargs)
)
@ -319,7 +319,7 @@ class RedisCache(BaseCache):
self.redis_version = "Unknown"
try:
if not coroutine_checker.is_async_callable(self.redis_client):
self.redis_version = self.redis_client.info()["redis_version"] # type: ignore
self.redis_version = self.redis_client.info()["redis_version"]
except Exception:
pass
@ -355,7 +355,7 @@ class RedisCache(BaseCache):
# SYNC HEALTH PING
try:
if hasattr(self.redis_client, "ping"):
self.redis_client.ping() # type: ignore
self.redis_client.ping()
except Exception as e:
verbose_logger.error("Error connecting to Sync Redis client", extra={"error": str(e)})
self._handle_sync_ping_error(e)
@ -423,7 +423,7 @@ class RedisCache(BaseCache):
redis_async_client = get_redis_async_client(connection_pool=self.async_redis_conn_pool, **self.redis_kwargs)
in_memory_llm_clients_cache.set_cache(key=cache_key, value=redis_async_client)
self.redis_async_client = redis_async_client # type: ignore
self.redis_async_client = redis_async_client
return redis_async_client
def check_and_fix_namespace(self, key: str) -> str:
@ -431,7 +431,7 @@ class RedisCache(BaseCache):
Make sure each key starts with the given namespace
"""
if key is None:
return key # type: ignore[return-value]
return key
if self.namespace is not None and not key.startswith(self.namespace):
key = self.namespace + ":" + key
@ -493,7 +493,7 @@ class RedisCache(BaseCache):
key = self.check_and_fix_namespace(key=key)
try:
start_time = time.time()
result: Final[int] = _redis_client.incr(name=key, amount=value) # type: ignore
result: Final[int] = _redis_client.incr(name=key, amount=value)
end_time = time.time()
_duration = end_time - start_time
self.service_logger_obj.service_success_hook(
@ -520,7 +520,7 @@ class RedisCache(BaseCache):
if current_ttl == -1:
# Key has no expiration
start_time = time.time()
_redis_client.expire(key, set_ttl) # type: ignore
_redis_client.expire(key, set_ttl)
end_time = time.time()
_duration = end_time - start_time
self.service_logger_obj.service_success_hook(
@ -555,7 +555,7 @@ class RedisCache(BaseCache):
return []
pattern = self.check_and_fix_namespace(key=pattern)
async for key in _redis_client.scan_iter(match=pattern + "*", count=count): # type: ignore
async for key in _redis_client.scan_iter(match=pattern + "*", count=count):
keys.append(key)
if len(keys) >= count:
break
@ -680,7 +680,7 @@ class RedisCache(BaseCache):
start_time: Final = time.time()
try:
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
except Exception as e:
end_time = time.time()
_duration = end_time - start_time
@ -773,7 +773,7 @@ class RedisCache(BaseCache):
_td: timedelta | None = None
if ttl is not None:
_td = timedelta(seconds=ttl)
pipe.set( # type: ignore
pipe.set(
name=cache_key,
value=json_cache_value,
ex=_td,
@ -849,7 +849,7 @@ class RedisCache(BaseCache):
"""Helper function for async_set_cache_sadd. Separated for testing."""
ttl = self.get_ttl(ttl=ttl)
try:
await redis_client.sadd(key, *value) # type: ignore
await redis_client.sadd(key, *value)
if ttl is not None:
_td: Final = timedelta(seconds=ttl)
await redis_client.expire(key, _td)
@ -862,7 +862,7 @@ class RedisCache(BaseCache):
start_time: Final = time.time()
try:
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
except Exception as e:
end_time = time.time()
_duration = end_time - start_time
@ -945,7 +945,7 @@ class RedisCache(BaseCache):
) -> float:
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
start_time: Final = time.time()
_used_ttl: Final = self.get_ttl(ttl=ttl)
key = self.check_and_fix_namespace(key=key)
@ -1080,7 +1080,7 @@ class RedisCache(BaseCache):
We use a wrapper so RedisCluster can override this method
"""
return self.redis_client.mget(keys=keys) # type: ignore
return self.redis_client.mget(keys=keys)
async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
"""
@ -1089,7 +1089,7 @@ class RedisCache(BaseCache):
We use a wrapper so RedisCluster can override this method
"""
async_redis_client: Final = self.init_async_client()
return await async_redis_client.mget(keys=keys) # type: ignore
return await async_redis_client.mget(keys=keys)
def batch_get_cache(
self,
@ -1147,7 +1147,7 @@ class RedisCache(BaseCache):
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
key = self.check_and_fix_namespace(key=key)
start_time: Final = time.time()
@ -1269,7 +1269,7 @@ class RedisCache(BaseCache):
print_verbose("Pinging Sync Redis Cache")
start_time: Final = time.time()
try:
response: Final[bool] = self.redis_client.ping() # type: ignore
response: Final[bool] = self.redis_client.ping()
print_verbose(f"Redis Cache PING: {response}")
## LOGGING ##
end_time = time.time()
@ -1339,7 +1339,7 @@ class RedisCache(BaseCache):
await _redis_client.delete(*keys)
def client_list(self) -> list:
client_list: Final[list] = self.redis_client.client_list() # type: ignore
client_list: Final[list] = self.redis_client.client_list()
return client_list
def info(self):
@ -1376,10 +1376,10 @@ class RedisCache(BaseCache):
redis_client: Final = redis_async.Redis(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping() # type: ignore[misc]
ping_result: Final = await redis_client.ping()
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]
await redis_client.aclose()
if ping_result:
return {
@ -1448,7 +1448,7 @@ class RedisCache(BaseCache):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client() # type: ignore
_redis_client: Final[Redis] = self.init_async_client()
start_time: Final = time.time()
print_verbose(f"Increment Async Redis Cache Pipeline: increment list: {increment_list}")
@ -1769,7 +1769,7 @@ class RedisCache(BaseCache):
or None
)
except Exception:
decoded_results.append(r) # type: ignore
decoded_results.append(r)
else:
decoded_results.append(None)
return decoded_results

14
litellm/caching/redis_cluster_cache.py
View file

@ -5,7 +5,7 @@ Key differences:
- RedisClient NEEDs to be re-used across requests, adds 3000ms latency if it's re-created
"""
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from litellm.caching.redis_cache import RedisCache
@ -16,7 +16,7 @@ if TYPE_CHECKING:
pipeline = Pipeline
async_redis_client = Redis
Span = Union[_Span, Any]
Span = _Span | Any
else:
pipeline = Any
async_redis_client = Any
@ -47,14 +47,14 @@ class RedisClusterCache(RedisCache):
"""
Overrides `_run_redis_mget_operation` in redis_cache.py
"""
return self.redis_client.mget_nonatomic(keys=keys) # type: ignore
return self.redis_client.mget_nonatomic(keys=keys)
async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
"""
Overrides `_async_run_redis_mget_operation` in redis_cache.py
"""
async_redis_cluster_client: Final = self.init_async_client()
return await async_redis_cluster_client.mget_nonatomic(keys=keys) # type: ignore
return await async_redis_cluster_client.mget_nonatomic(keys=keys)
async def test_connection(self) -> dict:
"""
@ -78,14 +78,14 @@ class RedisClusterCache(RedisCache):
# Create a fresh Redis Cluster client with current settings
redis_client: Final = redis_async.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs, # type: ignore
**cluster_kwargs,
)
# Test the connection
ping_result: Final = await redis_client.ping() # type: ignore[attr-defined, misc]
ping_result: Final = await redis_client.ping()
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]
await redis_client.aclose()
if ping_result:
return {

6
litellm/caching/redis_semantic_cache.py
View file

@ -126,8 +126,8 @@ class RedisSemanticCache(BaseCache):
# CustomTextVectorizer probes its embedding dimension at construction by
# embedding "dimension test", so the first cache request issues one extra
# billable embedding on top of the request's own.
from redisvl.extensions.llmcache import SemanticCache # type: ignore[import-not-found, import-untyped]
from redisvl.utils.vectorize import CustomTextVectorizer # type: ignore[import-not-found, import-untyped]
from redisvl.extensions.llmcache import SemanticCache
from redisvl.utils.vectorize import CustomTextVectorizer
try:
cache_vectorizer: Final = CustomTextVectorizer(self._get_embedding)
@ -207,7 +207,7 @@ class RedisSemanticCache(BaseCache):
return {self.CACHE_KEY_FIELD_NAME: str(key)}
def _get_cache_key_filter_expression(self, key: str) -> Any:
from redisvl.query.filter import Tag # type: ignore[import-not-found, import-untyped]
from redisvl.query.filter import Tag
return Tag(self.CACHE_KEY_FIELD_NAME) == str(key)

2
litellm/caching/s3_cache.py
View file

@ -146,7 +146,7 @@ class S3Cache(BaseCache):
)
return cached_response
except botocore.exceptions.ClientError as e: # type: ignore
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "NoSuchKey":
verbose_logger.debug("S3 Cache: The specified key '%s' does not exist in the S3 bucket.", key)
return None

8
litellm/caching/valkey_semantic_cache.py
View file

@ -85,12 +85,8 @@ class ValkeySemanticCache(RedisSemanticCache):
resolved_url = None
if sync_client is None or async_client is None:
resolved_url = redis_url or self._build_valkey_url(host, port, password, ssl)
self.sync_client = (
sync_client if sync_client is not None else Redis.from_url(resolved_url) # type: ignore[arg-type]
)
self.async_client = (
async_client if async_client is not None else AsyncRedis.from_url(resolved_url) # type: ignore[arg-type]
)
self.sync_client = sync_client if sync_client is not None else Redis.from_url(resolved_url)
self.async_client = async_client if async_client is not None else AsyncRedis.from_url(resolved_url)
print_verbose(f"Valkey semantic-cache initializing index - {self.index_name}")

4
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -238,7 +238,7 @@ class ResponsesToCompletionBridgeHandler:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
streaming_response=result,
sync_stream=True,
json_mode=kwargs.get("json_mode"),
)
@ -336,7 +336,7 @@ class ResponsesToCompletionBridgeHandler:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result, # type: ignore
streaming_response=result,
sync_stream=False,
json_mode=kwargs.get("json_mode"),
)

34
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -63,9 +63,9 @@ def _get_reasoning_items(
msg: "AllMessageValues",
) -> list[ChatCompletionReasoningItem]:
"""Extract reasoning_items from a message dict with proper typing."""
items: Final = msg.get("reasoning_items") # type: ignore[union-attr]
items: Final = msg.get("reasoning_items")
if items:
return items # type: ignore[return-value]
return items
return []
@ -261,8 +261,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content, # type: ignore[arg-type]
role, # type: ignore
content,
role,
),
}
)
@ -336,7 +336,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
{
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(content, cast(str, role)), # type: ignore[arg-type]
"content": self._convert_content_to_responses_format(content, cast(str, role)),
}
)
@ -360,17 +360,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif key == "response_format":
text_format = self._transform_response_format_to_text_format(value)
if text_format:
responses_api_request["text"] = text_format # type: ignore
responses_api_request["text"] = text_format
elif key == "tool_choice":
responses_api_request["tool_choice"] = ( # type: ignore[assignment]
self._normalize_tool_choice_for_responses_api(value)
)
responses_api_request["tool_choice"] = self._normalize_tool_choice_for_responses_api(value)
elif key == "stream_options":
stream_options = normalize_responses_api_stream_options(value)
if stream_options is not None:
responses_api_request["stream_options"] = stream_options
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key in ResponsesAPIOptionalRequestParams.__annotations__:
responses_api_request[key] = value
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
@ -524,7 +522,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
ResponseApplyPatchToolCall,
)
except ImportError:
ResponseApplyPatchToolCall = None # type: ignore[assignment,misc]
ResponseApplyPatchToolCall = None
from litellm.types.utils import Choices, Message
@ -942,7 +940,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
flat_custom["format"] = convert_custom_tool_format_to_responses_shape(custom_payload["format"])
responses_tools.append(flat_custom)
else:
responses_tools.append(tool) # type: ignore
responses_tools.append(tool)
return cast(list["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
@ -978,7 +976,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _map_reasoning_effort(self, reasoning_effort: str | dict[str, Any]) -> Reasoning | None:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(**reasoning_effort) # type: ignore[typeddict-item]
return Reasoning(**reasoning_effort)
# Check if auto-summary is enabled via flag or environment variable
# Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
@ -988,11 +986,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# If string is passed, map with optional summary based on flag/env var
if reasoning_effort == "none":
return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none") # type: ignore
return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
elif reasoning_effort == "high":
return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
elif reasoning_effort == "xhigh":
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh") # type: ignore[typeddict-item]
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
elif reasoning_effort == "medium":
return (
Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
@ -1108,7 +1106,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
verbose_logger.debug("Skipping unsupported annotation type: %s", type(annotation))
continue
result.append(annotation_dict) # type: ignore
result.append(annotation_dict)
except Exception as e:
# Skip malformed annotations
verbose_logger.debug("Skipping malformed annotation: %s, error: %s", annotation, e)
@ -1254,7 +1252,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
function=function_chunk,
)
if provider_specific_fields:
tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore
tool_call_chunk.provider_specific_fields = provider_specific_fields
return ModelResponseStream(
choices=[

2
litellm/compression/scoring/bm25.py
View file

@ -84,7 +84,7 @@ def bm25_score_messages(
# document tokens that start with that term (min 4 chars match). This lets
# "cook" match "cooking" and "auth" match "authentication" without a full
# stemmer dependency.
def _expand_tf(query_term: str, tf_counts: Counter) -> int: # type: ignore[type-arg]
def _expand_tf(query_term: str, tf_counts: Counter) -> int:
"""Sum TF across all doc tokens that are prefixed by query_term."""
exact: Final = tf_counts.get(query_term, 0)
if exact:

1
litellm/constants.py
View file

@ -1305,6 +1305,7 @@ RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when conv
########################### Logging Callback Constants ###########################
AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07"
AZURE_STORAGE_DEFAULT_ENDPOINT_SUFFIX: Final = "core.windows.net"
PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = int(
os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5)
)

14
litellm/containers/__init__.py
View file

@ -23,22 +23,20 @@ from .main import (
)
__all__ = [
# Core container operations
"acreate_container",
"adelete_container",
"alist_containers",
"aretrieve_container",
"create_container",
"delete_container",
"list_containers",
"retrieve_container",
# Container file operations (auto-generated from endpoints.json)
"adelete_container_file",
"alist_container_files",
"alist_containers",
"aretrieve_container",
"aretrieve_container_file",
"aretrieve_container_file_content",
"create_container",
"delete_container",
"delete_container_file",
"list_container_files",
"list_containers",
"retrieve_container",
"retrieve_container_file",
"retrieve_container_file_content",
]

12
litellm/containers/main.py
View file

@ -187,7 +187,7 @@ def create_container(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -405,7 +405,7 @@ def list_containers(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -596,7 +596,7 @@ def retrieve_container(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -811,7 +811,7 @@ def delete_container(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -1040,7 +1040,7 @@ def list_container_files(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True
@ -1291,7 +1291,7 @@ def upload_container_file(
local_vars: Final = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.pop("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id")
_is_async: Final = kwargs.pop("async_call", False) is True

6
litellm/containers/utils.py
View file

@ -53,7 +53,7 @@ class ContainerRequestUtils:
for param in valid_params:
if param in passed_params and passed_params[param] is not None:
container_create_optional_params[param] = passed_params[param] # type: ignore
container_create_optional_params[param] = passed_params[param]
return container_create_optional_params
@ -69,7 +69,7 @@ class ContainerRequestUtils:
filtered_params: Final = {k: v for k, v in container_create_optional_params.items() if k in supported_params}
return container_provider_config.map_openai_params(
container_create_optional_params=filtered_params, # type: ignore
container_create_optional_params=filtered_params,
drop_params=False,
)
@ -90,7 +90,7 @@ class ContainerRequestUtils:
for param in valid_params:
if param in passed_params and passed_params[param] is not None:
container_list_optional_params[param] = passed_params[param] # type: ignore
container_list_optional_params[param] = passed_params[param]
return container_list_optional_params

14
litellm/cost_calculator.py
View file

@ -329,7 +329,7 @@ def cost_per_token(
response: Any | None = None,
### REQUEST MODEL ###
request_model: str | None = None, # original request model for router detection
) -> tuple[float, float]: # type: ignore
) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -878,6 +878,8 @@ def _get_usage_object(
return None
if isinstance(usage_obj, Usage):
return usage_obj
elif isinstance(usage_obj, dict) and litellm.AnthropicConfig.is_anthropic_usage_object(usage_obj):
return litellm.AnthropicConfig().calculate_usage(usage_object=usage_obj, reasoning_content=None)
elif (
usage_obj is not None
and (isinstance(usage_obj, dict) or isinstance(usage_obj, ResponseAPIUsage))
@ -1249,7 +1251,13 @@ def completion_cost(
else:
_usage = usage_obj
if ResponseAPILoggingUtils._is_response_api_usage(_usage):
if litellm.AnthropicConfig.is_anthropic_usage_object(_usage):
_usage = (
litellm.AnthropicConfig()
.calculate_usage(usage_object=_usage, reasoning_content=None)
.model_dump()
)
elif ResponseAPILoggingUtils._is_response_api_usage(_usage):
_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
_usage
).model_dump()
@ -1514,7 +1522,7 @@ def completion_cost(
# see https://replicate.com/pricing
elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost:
# for unmapped replicate model, default to replicate's time tracking logic
return get_replicate_completion_pricing(completion_response, total_time) # type: ignore
return get_replicate_completion_pricing(completion_response, total_time)
if model is None:
raise ValueError(

88
litellm/evals/main.py
View file

@ -141,7 +141,7 @@ def create_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acreate_eval", False) is True
@ -153,7 +153,7 @@ def create_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -162,15 +162,15 @@ def create_eval(
# Build create request
create_request: Final[CreateEvalRequest] = {
"data_source_config": data_source_config, # type: ignore
"testing_criteria": testing_criteria, # type: ignore
"data_source_config": data_source_config,
"testing_criteria": testing_criteria,
}
if name is not None:
create_request["name"] = name
# Merge extra_body if provided
if extra_body:
create_request.update(extra_body) # type: ignore
create_request.update(extra_body)
# Validate environment and get headers
headers = extra_headers or {}
@ -199,7 +199,7 @@ def create_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.create_eval_handler( # type: ignore
response: Final = base_llm_http_handler.create_eval_handler(
url=url,
request_body=request_body,
evals_api_provider_config=evals_api_provider_config,
@ -326,7 +326,7 @@ def list_evals(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("alist_evals", False) is True
@ -338,7 +338,7 @@ def list_evals(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -354,13 +354,13 @@ def list_evals(
if before is not None:
list_params["before"] = before
if order is not None:
list_params["order"] = order # type: ignore
list_params["order"] = order
if order_by is not None:
list_params["order_by"] = order_by # type: ignore
list_params["order_by"] = order_by
# Merge extra_query if provided
if extra_query:
list_params.update(extra_query) # type: ignore
list_params.update(extra_query)
# Validate environment and get headers
headers = extra_headers or {}
@ -385,7 +385,7 @@ def list_evals(
)
# Make HTTP request
response: Final = base_llm_http_handler.list_evals_handler( # type: ignore
response: Final = base_llm_http_handler.list_evals_handler(
url=url,
query_params=query_params,
evals_api_provider_config=evals_api_provider_config,
@ -492,7 +492,7 @@ def get_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aget_eval", False) is True
@ -504,7 +504,7 @@ def get_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -536,7 +536,7 @@ def get_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.get_eval_handler( # type: ignore
response: Final = base_llm_http_handler.get_eval_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -657,7 +657,7 @@ def update_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aupdate_eval", False) is True
@ -669,7 +669,7 @@ def update_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -723,7 +723,7 @@ def update_eval(
# Merge extra_body if provided
if extra_body:
update_request.update(extra_body) # type: ignore
update_request.update(extra_body)
# Validate environment and get headers
headers = extra_headers or {}
@ -755,7 +755,7 @@ def update_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.update_eval_handler( # type: ignore
response: Final = base_llm_http_handler.update_eval_handler(
url=url,
request_body=request_body,
evals_api_provider_config=evals_api_provider_config,
@ -862,7 +862,7 @@ def delete_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("adelete_eval", False) is True
@ -874,7 +874,7 @@ def delete_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -906,7 +906,7 @@ def delete_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.delete_eval_handler( # type: ignore
response: Final = base_llm_http_handler.delete_eval_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1012,7 +1012,7 @@ def cancel_eval(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acancel_eval", False) is True
@ -1024,7 +1024,7 @@ def cancel_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1060,7 +1060,7 @@ def cancel_eval(
)
# Make HTTP request
response: Final = base_llm_http_handler.cancel_eval_handler( # type: ignore
response: Final = base_llm_http_handler.cancel_eval_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1191,7 +1191,7 @@ def create_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acreate_run", False) is True
@ -1203,7 +1203,7 @@ def create_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1212,7 +1212,7 @@ def create_run(
# Build create request
create_request: Final[CreateRunRequest] = {
"data_source": data_source, # type: ignore
"data_source": data_source,
}
if name is not None:
create_request["name"] = name
@ -1221,7 +1221,7 @@ def create_run(
# Merge extra_body if provided
if extra_body:
create_request.update(extra_body) # type: ignore
create_request.update(extra_body)
# Validate environment and get headers
headers = extra_headers or {}
@ -1248,7 +1248,7 @@ def create_run(
)
# Make HTTP request (default 600s timeout for long-running operations)
response: Final = base_llm_http_handler.create_run_handler( # type: ignore
response: Final = base_llm_http_handler.create_run_handler(
url=url,
request_body=request_body,
evals_api_provider_config=evals_api_provider_config,
@ -1375,7 +1375,7 @@ def list_runs(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("alist_runs", False) is True
@ -1387,7 +1387,7 @@ def list_runs(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1403,11 +1403,11 @@ def list_runs(
if before is not None:
list_params["before"] = before
if order is not None:
list_params["order"] = order # type: ignore
list_params["order"] = order
# Merge extra_query if provided
if extra_query:
list_params.update(extra_query) # type: ignore
list_params.update(extra_query)
# Validate environment and get headers
headers = extra_headers or {}
@ -1433,7 +1433,7 @@ def list_runs(
)
# Make HTTP request
response: Final = base_llm_http_handler.list_runs_handler( # type: ignore
response: Final = base_llm_http_handler.list_runs_handler(
url=url,
query_params=query_params,
evals_api_provider_config=evals_api_provider_config,
@ -1545,7 +1545,7 @@ def get_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("aget_run", False) is True
@ -1557,7 +1557,7 @@ def get_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1590,7 +1590,7 @@ def get_run(
)
# Make HTTP request
response: Final = base_llm_http_handler.get_run_handler( # type: ignore
response: Final = base_llm_http_handler.get_run_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1701,7 +1701,7 @@ def cancel_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("acancel_run", False) is True
@ -1713,7 +1713,7 @@ def cancel_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1750,7 +1750,7 @@ def cancel_run(
)
# Make HTTP request
response: Final = base_llm_http_handler.cancel_run_handler( # type: ignore
response: Final = base_llm_http_handler.cancel_run_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,
@ -1866,7 +1866,7 @@ def delete_run(
"""
local_vars: Final = locals()
try:
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
_is_async: Final = kwargs.pop("adelete_run", False) is True
@ -1878,7 +1878,7 @@ def delete_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
evals_api_provider_config: BaseEvalsAPIConfig | None = ProviderConfigManager.get_provider_evals_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
@ -1915,7 +1915,7 @@ def delete_run(
)
# Make HTTP request
response: Final = base_llm_http_handler.delete_run_handler( # type: ignore
response: Final = base_llm_http_handler.delete_run_handler(
url=url,
evals_api_provider_config=evals_api_provider_config,
custom_llm_provider=custom_llm_provider,

54
litellm/exceptions.py
View file

@ -126,7 +126,7 @@ def _get_minimal_error_response() -> httpx.Response:
return _MINIMAL_ERROR_RESPONSE
class AuthenticationError(openai.AuthenticationError): # type: ignore
class AuthenticationError(openai.AuthenticationError):
def __init__(
self,
message,
@ -170,7 +170,7 @@ class AuthenticationError(openai.AuthenticationError): # type: ignore
# raise when invalid models passed, example gpt-8
class NotFoundError(openai.NotFoundError): # type: ignore
class NotFoundError(openai.NotFoundError):
def __init__(
self,
message,
@ -213,7 +213,7 @@ class NotFoundError(openai.NotFoundError): # type: ignore
return _message
class BadRequestError(openai.BadRequestError): # type: ignore
class BadRequestError(openai.BadRequestError):
def __init__(
self,
message,
@ -288,7 +288,7 @@ class ImageFetchError(BadRequestError):
)
class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
message,
@ -327,7 +327,7 @@ class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore
return _message
class Timeout(openai.APITimeoutError): # type: ignore
class Timeout(openai.APITimeoutError):
def __init__(
self,
message,
@ -371,7 +371,7 @@ class Timeout(openai.APITimeoutError): # type: ignore
return _message
class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
class PermissionDeniedError(openai.PermissionDeniedError):
def __init__(
self,
message,
@ -410,7 +410,7 @@ class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
return _message
class RateLimitError(openai.RateLimitError): # type: ignore
class RateLimitError(openai.RateLimitError):
"""
Unified rate-limit error.
@ -501,7 +501,7 @@ class RateLimitError(openai.RateLimitError): # type: ignore
# sub class of rate limit error - meant to give more granularity for error handling context window exceeded errors
class ContextWindowExceededError(BadRequestError): # type: ignore
class ContextWindowExceededError(BadRequestError):
def __init__(
self,
message,
@ -516,8 +516,8 @@ class ContextWindowExceededError(BadRequestError): # type: ignore
self.litellm_debug_info = litellm_debug_info
super().__init__(
message=message,
model=self.model, # type: ignore
llm_provider=self.llm_provider, # type: ignore
model=self.model,
llm_provider=self.llm_provider,
response=response,
litellm_debug_info=self.litellm_debug_info,
) # Call the base class constructor with the parameters it needs
@ -543,7 +543,7 @@ class ContextWindowExceededError(BadRequestError): # type: ignore
# sub class of bad request error - meant to help us catch guardrails-related errors on proxy.
class RejectedRequestError(BadRequestError): # type: ignore
class RejectedRequestError(BadRequestError):
def __init__(
self,
message,
@ -562,8 +562,8 @@ class RejectedRequestError(BadRequestError): # type: ignore
response: Final = httpx.Response(status_code=400, request=request)
super().__init__(
message=self.message,
model=self.model, # type: ignore
llm_provider=self.llm_provider, # type: ignore
model=self.model,
llm_provider=self.llm_provider,
response=response,
litellm_debug_info=self.litellm_debug_info,
) # Call the base class constructor with the parameters it needs
@ -585,7 +585,7 @@ class RejectedRequestError(BadRequestError): # type: ignore
return _message
class ContentPolicyViolationError(BadRequestError): # type: ignore
class ContentPolicyViolationError(BadRequestError):
# Error code: 400 - {'error': {'code': 'content_policy_violation', 'message': 'Your request was rejected as a result of our safety system. Image descriptions generated from your prompt may contain text that is not allowed by our safety system. If you believe this was done in error, your request may succeed if retried, or by adjusting your prompt.', 'param': None, 'type': 'invalid_request_error'}}
def __init__(
self,
@ -605,8 +605,8 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore
self.provider_specific_fields = provider_specific_fields
super().__init__(
message=self.message,
model=self.model, # type: ignore
llm_provider=self.llm_provider, # type: ignore
model=self.model,
llm_provider=self.llm_provider,
response=response,
litellm_debug_info=self.litellm_debug_info,
body=body,
@ -630,7 +630,7 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore
return _message
class ServiceUnavailableError(openai.APIStatusError): # type: ignore
class ServiceUnavailableError(openai.APIStatusError):
def __init__(
self,
message,
@ -678,7 +678,7 @@ class ServiceUnavailableError(openai.APIStatusError): # type: ignore
return _message
class BadGatewayError(openai.APIStatusError): # type: ignore
class BadGatewayError(openai.APIStatusError):
def __init__(
self,
message,
@ -726,7 +726,7 @@ class BadGatewayError(openai.APIStatusError): # type: ignore
return _message
class InternalServerError(openai.InternalServerError): # type: ignore
class InternalServerError(openai.InternalServerError):
def __init__(
self,
message,
@ -775,7 +775,7 @@ class InternalServerError(openai.InternalServerError): # type: ignore
# raise this when the API returns an invalid response object - https://github.com/openai/openai-python/blob/1be14ee34a0f8e42d3f9aa5451aa4cb161f1781f/openai/api_requestor.py#L401
class APIError(openai.APIError): # type: ignore
class APIError(openai.APIError):
def __init__(
self,
status_code: int,
@ -796,7 +796,7 @@ class APIError(openai.APIError): # type: ignore
self.num_retries = num_retries
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=None) # type: ignore
super().__init__(self.message, request=request, body=None)
def __str__(self):
_message = self.message
@ -816,7 +816,7 @@ class APIError(openai.APIError): # type: ignore
# raised if an invalid request (not get, delete, put, post) is made
class APIConnectionError(openai.APIConnectionError): # type: ignore
class APIConnectionError(openai.APIConnectionError):
def __init__(
self,
message,
@ -855,7 +855,7 @@ class APIConnectionError(openai.APIConnectionError): # type: ignore
# raised if an invalid request (not get, delete, put, post) is made
class APIResponseValidationError(openai.APIResponseValidationError): # type: ignore
class APIResponseValidationError(openai.APIResponseValidationError):
def __init__(
self,
message,
@ -902,7 +902,7 @@ class JSONSchemaValidationError(APIResponseValidationError):
super().__init__(model=model, message=message, llm_provider=llm_provider)
class OpenAIError(openai.OpenAIError): # type: ignore
class OpenAIError(openai.OpenAIError):
def __init__(self, original_exception=None):
super().__init__()
self.llm_provider = "openai"
@ -987,7 +987,7 @@ class BudgetExceededError(Exception):
## DEPRECATED ##
class InvalidRequestError(openai.BadRequestError): # type: ignore
class InvalidRequestError(openai.BadRequestError):
def __init__(self, message, model, llm_provider):
self.status_code = 400
self.message = message
@ -1024,7 +1024,7 @@ class MockException(openai.APIError):
self.num_retries = num_retries
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=None) # type: ignore
super().__init__(self.message, request=request, body=None)
class LiteLLMUnknownProvider(BadRequestError):
@ -1070,7 +1070,7 @@ class BlockedPiiEntityError(Exception):
super().__init__(self.message)
class MidStreamFallbackError(ServiceUnavailableError): # type: ignore
class MidStreamFallbackError(ServiceUnavailableError):
def __init__(
self,
message: str,

2
litellm/experimental_mcp_client/client.py
View file

@ -15,7 +15,7 @@ from mcp.client.stdio import stdio_client
streamable_http_client: Any | None = None
try:
import mcp.client.streamable_http as streamable_http_module # type: ignore
import mcp.client.streamable_http as streamable_http_module
streamable_http_client = getattr(streamable_http_module, "streamable_http_client", None)
except ImportError:

32
litellm/files/main.py
View file

@ -131,7 +131,7 @@ async def acreate_file(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -176,7 +176,7 @@ def create_file(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -252,7 +252,7 @@ def create_file(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_file", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_file", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -328,7 +328,7 @@ def file_retrieve(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -419,7 +419,7 @@ def file_retrieve(
request=httpx.Request(
method="create_thread",
url="https://github.com/BerriAI/litellm",
), # type: ignore
),
),
)
@ -465,9 +465,9 @@ async def afile_delete(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return cast(FileDeleted, response) # type: ignore
return cast(FileDeleted, response)
except Exception as e:
raise e
@ -511,7 +511,7 @@ def file_delete(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
_is_async: Final = kwargs.pop("is_async", False) is True
@ -596,7 +596,7 @@ def file_delete(
request=httpx.Request(
method="create_thread",
url="https://github.com/BerriAI/litellm",
), # type: ignore
),
),
)
return cast(FileDeleted, response)
@ -639,7 +639,7 @@ async def afile_list(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -673,7 +673,7 @@ def file_list(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -755,7 +755,7 @@ def file_list(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="file_list", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="file_list", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -803,7 +803,7 @@ async def afile_content(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
@ -857,7 +857,7 @@ def file_content(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -987,7 +987,7 @@ def file_content(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -1065,7 +1065,7 @@ def file_content_streaming(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)

6
litellm/files/streaming.py
View file

@ -93,9 +93,9 @@ class FileContentStreamingResponse:
# are released promptly on client disconnects.
with anyio.CancelScope(shield=True):
if hasattr(stream_to_close, "aclose"):
await cast(AsyncIterator[bytes], stream_to_close).aclose() # type: ignore[attr-defined]
await cast(AsyncIterator[bytes], stream_to_close).aclose()
elif hasattr(stream_to_close, "close"):
result: Final = cast(Iterator[bytes], stream_to_close).close() # type: ignore[attr-defined]
result: Final = cast(Iterator[bytes], stream_to_close).close()
if result is not None:
await result
@ -109,7 +109,7 @@ class FileContentStreamingResponse:
self.stream_iterator = cast(Iterator[bytes] | AsyncIterator[bytes], iter(()))
if hasattr(stream_to_close, "close"):
cast(Iterator[bytes], stream_to_close).close() # type: ignore[attr-defined]
cast(Iterator[bytes], stream_to_close).close()
def _build_logging_response(self) -> dict[str, str]:
response: Final = {

48
litellm/fine_tuning/main.py
View file

@ -80,7 +80,7 @@ async def acreate_fine_tuning_job(
hyperparameters: dict | None = {},
suffix: str | None = None,
validation_file: str | None = None,
integrations: List[str] | None = None,
integrations: list[str] | None = None,
seed: int | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: dict[str, str] | None = None,
@ -119,7 +119,7 @@ async def acreate_fine_tuning_job(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -157,7 +157,7 @@ def create_fine_tuning_job(
hyperparameters: dict | None = {},
suffix: str | None = None,
validation_file: str | None = None,
integrations: List[str] | None = None,
integrations: list[str] | None = None,
seed: int | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: dict[str, str] | None = None,
@ -242,9 +242,9 @@ def create_fine_tuning_job(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -252,7 +252,7 @@ def create_fine_tuning_job(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -321,7 +321,7 @@ def create_fine_tuning_job(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -362,7 +362,7 @@ async def acancel_fine_tuning_job(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -396,7 +396,7 @@ def cancel_fine_tuning_job(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -441,7 +441,7 @@ def cancel_fine_tuning_job(
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -449,7 +449,7 @@ def cancel_fine_tuning_job(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -473,7 +473,7 @@ def cancel_fine_tuning_job(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -514,7 +514,7 @@ async def alist_fine_tuning_jobs(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -550,7 +550,7 @@ def list_fine_tuning_jobs(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -594,9 +594,9 @@ def list_fine_tuning_jobs(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -604,7 +604,7 @@ def list_fine_tuning_jobs(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -629,7 +629,7 @@ def list_fine_tuning_jobs(
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),
),
)
return response
@ -669,7 +669,7 @@ async def aretrieve_fine_tuning_job(
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response # type: ignore
response = init_response
return response
except Exception as e:
raise e
@ -700,7 +700,7 @@ def retrieve_fine_tuning_job(
read_timeout: Final = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
timeout = float(timeout)
elif timeout is None:
timeout = 600.0
@ -733,9 +733,9 @@ def retrieve_fine_tuning_job(
)
# Azure OpenAI
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
api_key = (
optional_params.api_key
@ -743,7 +743,7 @@ def retrieve_fine_tuning_job(
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
)
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
@ -770,7 +770,7 @@ def retrieve_fine_tuning_job(
request=httpx.Request(
method="retrieve_fine_tuning_job",
url="https://github.com/BerriAI/litellm",
), # type: ignore
),
),
)
return response

8
litellm/google_genai/main.py
View file

@ -156,7 +156,7 @@ class GenerateContentHelper:
model=model,
custom_llm_provider=custom_llm_provider,
request_body={}, # Will be handled by adapter
generate_content_provider_config=None, # type: ignore
generate_content_provider_config=None,
generate_content_config_dict=dict(config or {}),
native_request_fields={},
litellm_params=litellm_params,
@ -350,7 +350,7 @@ def generate_content(
# Use the adapter to convert to completion format
return GenerateContentToCompletionHandler.generate_content_handler(
model=model,
contents=contents, # type: ignore
contents=contents,
config=setup_result.generate_content_config_dict,
tools=tools,
_is_async=_is_async,
@ -444,7 +444,7 @@ async def agenerate_content_stream(
# Use the adapter to convert to completion format
return await GenerateContentToCompletionHandler.async_generate_content_handler(
model=model,
contents=contents, # type: ignore
contents=contents,
config=setup_result.generate_content_config_dict,
litellm_params=setup_result.litellm_params,
tools=tools,
@ -534,7 +534,7 @@ def generate_content_stream(
# Use the adapter to convert to completion format
return GenerateContentToCompletionHandler.generate_content_handler(
model=model,
contents=contents, # type: ignore
contents=contents,
config=setup_result.generate_content_config_dict,
_is_async=_is_async,
litellm_params=setup_result.litellm_params,

20
litellm/images/main.py
View file

@ -28,7 +28,7 @@ from litellm.utils import exception_type, get_litellm_params
#################### Initialize provider clients ####################
llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
from openai.types.audio.transcription_create_params import FileTypes # type: ignore
from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
@ -112,7 +112,7 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
elif isinstance(init_response, ImageResponse): ## CACHING SCENARIO
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
response = await init_response
if response is None:
raise ValueError("Unable to get Image Response. Please pass a valid llm_provider.")
@ -207,12 +207,12 @@ def image_generation(
aimg_generation: Final = kwargs.get("aimg_generation", False)
litellm_call_id: Final = kwargs.get("litellm_call_id", None)
logger_fn: Final = kwargs.get("logger_fn", None)
mock_response: Final[str | None] = kwargs.get("mock_response", None) # type: ignore
mock_response: Final[str | None] = kwargs.get("mock_response", None)
proxy_server_request: Final = kwargs.get("proxy_server_request", None)
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
model_info: Final = kwargs.get("model_info", None)
metadata: Final = kwargs.get("metadata", {})
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
client: Final = kwargs.get("client", None)
extra_headers: Final = kwargs.get("extra_headers", None)
headers: Final[dict] = kwargs.get("headers", None) or {}
@ -223,7 +223,7 @@ def image_generation(
dynamic_api_key: str | None = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, # type: ignore
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
@ -479,7 +479,7 @@ def image_generation(
elif custom_llm_provider == "bedrock":
if model is None:
raise Exception("Model needs to be set for bedrock")
model_response = bedrock_image_generation.image_generation( # type: ignore
model_response = bedrock_image_generation.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
@ -508,7 +508,7 @@ def image_generation(
async_custom_client = client
## CALL FUNCTION
model_response = custom_handler.aimage_generation( # type: ignore
model_response = custom_handler.aimage_generation(
model=model,
prompt=prompt,
api_key=api_key,
@ -584,7 +584,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
init_response = ImageResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
response = await init_response # type: ignore
response = await init_response
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
@ -745,7 +745,7 @@ def image_edit(
non_default_params: Final = {
k: v for k, v in kwargs.items() if k not in default_params
} # model-specific params - pass them straight to the model/provider
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") # type: ignore
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None)
model_info: Final = kwargs.get("model_info", None)
metadata: Final = kwargs.get("metadata", {})
@ -860,7 +860,7 @@ def image_edit(
if model is None:
raise Exception("Model needs to be set for bedrock")
image_edit_request_params.update(non_default_params)
return bedrock_image_edit.image_edit( # type: ignore
return bedrock_image_edit.image_edit(
model=model,
image=images,
prompt=prompt,

30
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -709,7 +709,7 @@ class SlackAlerting(CustomBatchLogger):
"""Format an alert message for slack"""
headers: Final = {f"{key} Name": key_val, "Provider": provider}
if api_base is not None:
headers["API Base"] = api_base # type: ignore
headers["API Base"] = api_base
headers_str = "\n"
for k, v in headers.items():
@ -767,14 +767,11 @@ class SlackAlerting(CustomBatchLogger):
# Convert deployment_ids back to set if it was stored as a list
if outage_value is not None:
outage_value = self._restore_outage_value_from_cache(outage_value) # type: ignore
outage_value = self._restore_outage_value_from_cache(outage_value)
if (
getattr(exception, "status_code", None) is None
or (
exception.status_code != 408 # type: ignore
and exception.status_code < 500 # type: ignore
)
or (exception.status_code != 408 and exception.status_code < 500)
or self.llm_router is None
):
return
@ -784,7 +781,7 @@ class SlackAlerting(CustomBatchLogger):
_deployment_set.add(deployment_id)
outage_value = ProviderRegionOutageModel(
provider_region_id=cache_key,
alerts=[exception.status_code], # type: ignore
alerts=[exception.status_code],
minor_alert_sent=False,
major_alert_sent=False,
last_updated_at=time.time(),
@ -802,7 +799,7 @@ class SlackAlerting(CustomBatchLogger):
return
if len(outage_value["alerts"]) < self.alerting_args.max_outage_alert_list_size:
outage_value["alerts"].append(exception.status_code) # type: ignore
outage_value["alerts"].append(exception.status_code)
else: # prevent memory leaks
pass
_deployment_set = outage_value["deployment_ids"]
@ -884,13 +881,10 @@ class SlackAlerting(CustomBatchLogger):
max_alerts_size = 10
"""
try:
outage_value: OutageModel | None = await self.internal_usage_cache.async_get_cache(key=deployment_id) # type: ignore
outage_value: OutageModel | None = await self.internal_usage_cache.async_get_cache(key=deployment_id)
if (
getattr(exception, "status_code", None) is None
or (
exception.status_code != 408 # type: ignore
and exception.status_code < 500 # type: ignore
)
or (exception.status_code != 408 and exception.status_code < 500)
or self.llm_router is None
):
return
@ -912,7 +906,7 @@ class SlackAlerting(CustomBatchLogger):
if outage_value is None:
outage_value = OutageModel(
model_id=deployment_id,
alerts=[exception.status_code], # type: ignore
alerts=[exception.status_code],
minor_alert_sent=False,
major_alert_sent=False,
last_updated_at=time.time(),
@ -927,7 +921,7 @@ class SlackAlerting(CustomBatchLogger):
return
if len(outage_value["alerts"]) < self.alerting_args.max_outage_alert_list_size:
outage_value["alerts"].append(exception.status_code) # type: ignore
outage_value["alerts"].append(exception.status_code)
else: # prevent memory leaks
pass
@ -1483,10 +1477,10 @@ Model Info:
if isinstance(response_obj, litellm.ModelResponse) and (
hasattr(response_obj, "usage")
and response_obj.usage is not None # type: ignore
and hasattr(response_obj.usage, "completion_tokens") # type: ignore
and response_obj.usage is not None
and hasattr(response_obj.usage, "completion_tokens")
):
completion_tokens: Final = response_obj.usage.completion_tokens # type: ignore
completion_tokens: Final = response_obj.usage.completion_tokens
if completion_tokens is not None and completion_tokens > 0:
final_value = float(response_s.total_seconds() / completion_tokens)
if isinstance(final_value, timedelta):

4
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -225,11 +225,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# 1. if string, insert cache control in the message
if isinstance(message_content, str):
message["cache_control"] = control # type: ignore
message["cache_control"] = control
# 2. list of objects - only apply to last item per Anthropic spec
elif isinstance(message_content, list):
if len(message_content) > 0 and isinstance(message_content[-1], dict):
message_content[-1]["cache_control"] = control # type: ignore
message_content[-1]["cache_control"] = control
return message
@staticmethod

10
litellm/integrations/argilla.py
View file

@ -10,7 +10,7 @@ import types
from typing import Any, Final
import httpx
from pydantic import BaseModel # type: ignore
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
@ -56,8 +56,8 @@ class ArgillaLogger(CustomBatchLogger):
argilla_base_url=argilla_base_url,
)
self.sampling_rate: float = (
float(os.getenv("ARGILLA_SAMPLING_RATE")) # type: ignore
if os.getenv("ARGILLA_SAMPLING_RATE") is not None and os.getenv("ARGILLA_SAMPLING_RATE").strip().isdigit() # type: ignore
float(os.getenv("ARGILLA_SAMPLING_RATE"))
if os.getenv("ARGILLA_SAMPLING_RATE") is not None and os.getenv("ARGILLA_SAMPLING_RATE").strip().isdigit()
else 1.0
)
@ -196,9 +196,9 @@ class ArgillaLogger(CustomBatchLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate: Final = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
float(os.getenv("LANGSMITH_SAMPLING_RATE"))
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit()
else 1.0
)
random_sample: Final = random.random()

4
litellm/integrations/arize/arize.py
View file

@ -6,7 +6,7 @@ this file has Arize ai specific helper functions
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from litellm.integrations.arize import _utils
from litellm.integrations.arize._utils import ArizeOTELAttributes
@ -21,7 +21,7 @@ if TYPE_CHECKING:
from litellm.types.integrations.arize import Protocol as _Protocol
Protocol = _Protocol
Span = Union[_Span, Any]
Span = _Span | Any
else:
Protocol = Any
Span = Any

10
litellm/integrations/arize/arize_phoenix.py
View file

@ -1,7 +1,7 @@
import os
import threading
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Final, Union
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
@ -22,7 +22,7 @@ if TYPE_CHECKING:
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
Span = _Span | Any
OpenTelemetry = _OpenTelemetry
LITELLM_TRACER_NAME: str
else:
@ -40,14 +40,14 @@ else:
)
except ImportError:
LITELLM_TRACER_NAME = "litellm"
OpenTelemetry = None # type: ignore
OpenTelemetry = None
ARIZE_HOSTED_PHOENIX_ENDPOINT: Final = "https://otlp.arize.com/v1/traces"
_MAX_PROJECT_PROVIDERS: Final = 64
class ArizePhoenixLogger(OpenTelemetry): # type: ignore
class ArizePhoenixLogger(OpenTelemetry):
"""
Arize Phoenix logger that sends traces to a Phoenix endpoint.
@ -139,7 +139,7 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
project_attributes["deployment.environment"] = deployment_environment
env_resource: Final = OTELResourceDetector().detect()
project_resource: Final = Resource.create(project_attributes) # type: ignore[arg-type]
project_resource: Final = Resource.create(project_attributes)
return env_resource.merge(project_resource)
def _build_tracer_provider_for_project(self, project_name: str) -> TracerProvider:

4
litellm/integrations/arize/arize_phoenix_prompt_manager.py
View file

@ -174,9 +174,7 @@ class ArizePhoenixTemplateManager:
# Combine rendered content
final_content = " ".join(rendered_content_parts)
rendered_messages.append(
{"role": role, "content": final_content} # type: ignore
)
rendered_messages.append({"role": role, "content": final_content})
return rendered_messages

21
litellm/integrations/azure_storage/azure_storage.py
View file

@ -6,7 +6,11 @@ from typing import Final
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS, AZURE_STORAGE_MSFT_VERSION
from litellm.constants import (
_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
AZURE_STORAGE_DEFAULT_ENDPOINT_SUFFIX,
AZURE_STORAGE_MSFT_VERSION,
)
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
@ -41,6 +45,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
if not _azure_storage_file_system:
raise ValueError("Missing required environment variable: AZURE_STORAGE_FILE_SYSTEM")
self.azure_storage_file_system: str = _azure_storage_file_system
self.azure_storage_endpoint_suffix: str = (
os.getenv("AZURE_STORAGE_ENDPOINT_SUFFIX") or AZURE_STORAGE_DEFAULT_ENDPOINT_SUFFIX
)
self._service_client = None
# Time that the azure service client expires, in order to reset the connection pool and keep it fresh
self._service_client_timeout: float | None = None
@ -59,6 +66,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
)
raise e
@property
def azure_storage_dfs_endpoint(self) -> str:
return f"https://{self.azure_storage_account_name}.dfs.{self.azure_storage_endpoint_suffix}"
@property
def azure_storage_blob_endpoint(self) -> str:
return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""
Async Log success events to Azure Blob Storage
@ -144,7 +159,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry
payload_bytes: Final = json_payload.encode("utf-8")
filename: Final = f"{payload.get('id') or str(uuid.uuid4())}.json"
base_url = f"https://{self.azure_storage_account_name}.dfs.core.windows.net/{self.azure_storage_file_system}/{filename}"
base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
# Execute the 3-step upload process
await self._create_file(async_client, base_url)
@ -296,7 +311,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client = None
if not self._service_client:
self._service_client = DataLakeServiceClient(
account_url=f"https://{self.azure_storage_account_name}.dfs.core.windows.net",
account_url=self.azure_storage_dfs_endpoint,
credential=self.azure_storage_account_key,
)
self._service_client_timeout = time.time() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS

2
litellm/integrations/bitbucket/__init__.py
View file

@ -27,7 +27,7 @@ def set_global_bitbucket_config(config: dict) -> None:
"""
import litellm
litellm.global_bitbucket_config = config # type: ignore
litellm.global_bitbucket_config = config
def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec") -> "CustomPromptManagement":

14
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -292,9 +292,7 @@ class BitBucketPromptManager(CustomPromptManagement):
final_messages: list[AllMessageValues] = parsed_messages
else:
# If no messages were parsed, prepend the prompt to existing messages
final_messages = [
{"role": "user", "content": rendered_prompt} # type: ignore
] + messages
final_messages = [{"role": "user", "content": rendered_prompt}] + messages
# Update litellm_params with prompt metadata
if litellm_params is None:
@ -345,7 +343,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
} # type: ignore
}
)
current_role = "system"
current_content = [line[7:].strip()] # Remove "System:" prefix
@ -355,7 +353,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
} # type: ignore
}
)
current_role = "user"
current_content = [line[5:].strip()] # Remove "User:" prefix
@ -365,7 +363,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
} # type: ignore
}
)
current_role = "assistant"
current_content = [line[10:].strip()] # Remove "Assistant:" prefix
@ -379,9 +377,9 @@ class BitBucketPromptManager(CustomPromptManagement):
# If no role indicators found, treat as a single user message
if not messages and prompt_content.strip():
messages = [{"role": "user", "content": prompt_content.strip()}] # type: ignore
messages = [{"role": "user", "content": prompt_content.strip()}]
return messages # type: ignore
return messages
def post_call_hook(
self,

6
litellm/integrations/braintrust_logging.py
View file

@ -28,9 +28,9 @@ def get_utc_datetime():
import datetime as dt
if hasattr(dt, "UTC"):
return datetime.now(dt.UTC) # type: ignore
return datetime.now(dt.UTC)
else:
return datetime.utcnow() # type: ignore
return datetime.utcnow()
class BraintrustLogger(CustomLogger):
@ -43,7 +43,7 @@ class BraintrustLogger(CustomLogger):
self.validate_environment(api_key=api_key)
self.api_base = api_base or os.getenv("BRAINTRUST_API_BASE") or API_BASE
self.default_project_id = None
self.api_key: str = api_key or os.getenv("BRAINTRUST_API_KEY") # type: ignore
self.api_key: str = api_key or os.getenv("BRAINTRUST_API_KEY")
self.headers = {
"Authorization": "Bearer " + self.api_key,
"Content-Type": "application/json",

2
litellm/integrations/braintrust_mock_client.py
View file

@ -150,7 +150,7 @@ def create_mock_braintrust_client():
if _original_http_handler_post is None:
_original_http_handler_post = HTTPHandler.post
HTTPHandler.post = _mock_http_handler_post # type: ignore
HTTPHandler.post = _mock_http_handler_post
verbose_logger.debug("[BRAINTRUST MOCK] Patched HTTPHandler.post")
# CRITICAL: Call the factory's initialization function to patch AsyncHTTPHandler.post

2
litellm/integrations/compression_interception/handler.py
View file

@ -133,7 +133,7 @@ class CompressionInterceptionLogger(CustomLogger):
self._prune_expired_cache()
compressed: Final = compress( # type: ignore
compressed: Final = compress(
messages=messages,
model=model,
call_type=CallTypes.anthropic_messages,

8
litellm/integrations/custom_guardrail.py
View file

@ -34,7 +34,7 @@ from litellm.types.utils import (
try:
from fastapi.exceptions import HTTPException
except ImportError:
HTTPException = None # type: ignore
HTTPException = None
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -410,7 +410,7 @@ class CustomGuardrail(CustomLogger):
if self.should_route_on_sensitive_data():
try:
self.raise_sensitive_data_route_exception(
route_to_model=self.sensitive_data_route_to_model, # type: ignore
route_to_model=self.sensitive_data_route_to_model,
request_data=request_data,
detection_info=detection_info,
)
@ -892,9 +892,9 @@ class CustomGuardrail(CustomLogger):
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump())) # type: ignore[typeddict-item]
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
else:
guardrail_mode = self.event_hook # type: ignore[assignment]
guardrail_mode = self.event_hook
from litellm.litellm_core_utils.core_helpers import (
filter_exceptions_from_params,

12
litellm/integrations/custom_logger.py
View file

@ -3,7 +3,7 @@
import re
import traceback
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any, Final, Optional, Union
from typing import TYPE_CHECKING, Any, Final, Optional
from pydantic import BaseModel
@ -39,7 +39,7 @@ if TYPE_CHECKING:
)
from litellm.types.router import PreRoutingHookResponse
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any
LiteLLMLoggingObj = Any
@ -783,13 +783,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
- Converting to string and then truncating the logged content catches this
2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
"""
field_value: Final = standard_logging_object.get(field_name) # type: ignore
field_value: Final = standard_logging_object.get(field_name)
if field_value:
str_value: Final = str(field_value)
if len(str_value) > max_length:
standard_logging_object[field_name] = self._truncate_text( # type: ignore
text=str_value, max_length=max_length
)
standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
def _truncate_text(self, text: str, max_length: int) -> str:
"""Truncate text if it exceeds max_length"""
@ -911,7 +909,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
for callback_obj in all_callbacks:
if hasattr(callback_obj, "increment_callback_logging_failure"):
verbose_logger.debug("Incrementing callback failure metric for %s", callback_name)
callback_obj.increment_callback_logging_failure(callback_name=callback_name) # type: ignore
callback_obj.increment_callback_logging_failure(callback_name=callback_name)
return
verbose_logger.debug(

4
litellm/integrations/datadog/datadog.py
View file

@ -500,7 +500,7 @@ class DataDogLogger(
response: Final = self.sync_client.post(
url=self.intake_url,
json=dd_payload, # type: ignore
json=dd_payload,
headers=headers,
)
@ -616,7 +616,7 @@ class DataDogLogger(
response: Final = await self.async_client.post(
url=self.intake_url,
data=compressed_data, # type: ignore
data=compressed_data,
headers=headers,
)
return response

8
litellm/integrations/datadog/datadog_metrics.py
View file

@ -91,9 +91,9 @@ class DatadogMetricsLogger(CustomBatchLogger):
metadata: Final = log.get("metadata", {}) or {}
team_tag: Final = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias") # type: ignore
or metadata.get("team_alias")
or metadata.get("user_api_key_team_id")
or metadata.get("team_id") # type: ignore
or metadata.get("team_id")
)
if team_tag:
@ -193,7 +193,7 @@ class DatadogMetricsLogger(CustomBatchLogger):
# Extract status code from error information
status_code = "500" # default
error_information: Final = standard_logging_object.get("error_information", {}) or {}
error_code: Final = error_information.get("error_code") # type: ignore
error_code: Final = error_information.get("error_code")
if error_code is not None:
status_code = str(error_code)
@ -237,7 +237,7 @@ class DatadogMetricsLogger(CustomBatchLogger):
response: Final = await self.async_client.post(
self.upload_url,
content=compressed_data,
headers=headers, # type: ignore
headers=headers,
)
response.raise_for_status()

2
litellm/integrations/dotprompt/__init__.py
View file

@ -24,7 +24,7 @@ def set_global_prompt_directory(directory: str) -> None:
"""
import litellm
litellm.global_prompt_directory = directory # type: ignore
litellm.global_prompt_directory = directory
def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict:

2
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -311,7 +311,7 @@ class DotpromptManager(CustomPromptManagement):
def _create_message(self, role: str, content: str) -> AllMessageValues:
"""Create a message with the specified role and content."""
return {
"role": role, # type: ignore
"role": role,
"content": content,
}

4
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -31,7 +31,7 @@ class PromptTemplate:
self.output_format = self.metadata.get("output", {}).get("format")
self.output_schema = self.metadata.get("output", {}).get("schema", {})
self.optional_params = {}
for key in self.metadata.keys():
for key in self.metadata:
if key not in restricted_keys:
self.optional_params[key] = self.metadata[key]
@ -253,7 +253,7 @@ class PromptManager:
"dict": dict,
}
return type_mapping.get(schema_type.lower(), str) # type: ignore
return type_mapping.get(schema_type.lower(), str)
def get_prompt(self, prompt_id: str, version: int | None = None) -> PromptTemplate | None:
"""

4
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -42,9 +42,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
batch_size=self.batch_size,
flush_interval=self.flush_interval,
)
self.log_queue: asyncio.Queue[GCSLogQueueItem] = asyncio.Queue( # type: ignore[assignment]
maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE
)
self.log_queue: asyncio.Queue[GCSLogQueueItem] = asyncio.Queue(maxsize=LITELLM_ASYNCIO_QUEUE_MAXSIZE)
asyncio.create_task(self.periodic_flush())
AdditionalLoggingUtils.__init__(self)

10
litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
View file

@ -167,12 +167,12 @@ def create_mock_gcs_client():
if _original_async_handler_get is None:
_original_async_handler_get = AsyncHTTPHandler.get
AsyncHTTPHandler.get = _mock_async_handler_get # type: ignore
AsyncHTTPHandler.get = _mock_async_handler_get
verbose_logger.debug("[GCS MOCK] Patched AsyncHTTPHandler.get")
if _original_async_handler_delete is None:
_original_async_handler_delete = AsyncHTTPHandler.delete
AsyncHTTPHandler.delete = _mock_async_handler_delete # type: ignore
AsyncHTTPHandler.delete = _mock_async_handler_delete
verbose_logger.debug("[GCS MOCK] Patched AsyncHTTPHandler.delete")
verbose_logger.debug(f"[GCS MOCK] Mock latency set to {_MOCK_LATENCY_SECONDS * 1000:.0f}ms")
@ -227,9 +227,9 @@ def mock_vertex_auth_methods():
return ("mock-gcs-token", "https://storage.googleapis.com")
# Patch the methods
VertexBase._ensure_access_token_async = _mock_ensure_access_token_async # type: ignore
VertexBase._ensure_access_token = _mock_ensure_access_token # type: ignore
VertexBase._get_token_and_url = _mock_get_token_and_url # type: ignore
VertexBase._ensure_access_token_async = _mock_ensure_access_token_async
VertexBase._ensure_access_token = _mock_ensure_access_token
VertexBase._get_token_and_url = _mock_get_token_and_url
verbose_logger.debug("[GCS MOCK] Patched Vertex AI auth methods")

2
litellm/integrations/generic_api/generic_api_callback.py
View file

@ -382,7 +382,7 @@ class GenericAPILogger(CustomBatchLogger):
verbose_logger.debug(
"Generic API Logger - sent log %s, status: %s",
idx,
result.status_code, # type: ignore
result.status_code,
)
else:
# Format the payload based on log_format

2
litellm/integrations/generic_prompt_management/__init__.py
View file

@ -28,7 +28,7 @@ def set_global_generic_prompt_config(config: dict) -> None:
"""
import litellm
litellm.global_generic_prompt_config = config # type: ignore
litellm.global_generic_prompt_config = config
def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec") -> "CustomPromptManagement":

4
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -366,14 +366,14 @@ class GenericPromptManager(CustomPromptManagement):
# Create a copy of the prompt template with variables applied
updated_messages: Final[list[AllMessageValues]] = []
for message in prompt_client["prompt_template"]:
updated_message = dict(message) # type: ignore
updated_message = dict(message)
if "content" in updated_message and isinstance(updated_message["content"], str):
content = updated_message["content"]
for key, value in variables.items():
content = content.replace(f"{{{key}}}", str(value))
content = content.replace(f"{{{{{key}}}}}", str(value)) # Also support {{key}}
updated_message["content"] = content
updated_messages.append(updated_message) # type: ignore
updated_messages.append(updated_message)
return PromptManagementClient(
prompt_id=prompt_client["prompt_id"],

2
litellm/integrations/gitlab/__init__.py
View file

@ -28,7 +28,7 @@ def set_global_gitlab_config(config: dict) -> None:
"""
import litellm
litellm.global_gitlab_config = config # type: ignore
litellm.global_gitlab_config = config
def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec") -> "CustomPromptManagement":

14
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -257,7 +257,7 @@ class GitLabTemplateManager:
and str(f.get("path", "")).endswith(".prompt")
and "path" in f
):
files.append(f["path"]) # type: ignore
files.append(f["path"])
return [self._repo_path_to_id(p) for p in files]
@ -357,7 +357,7 @@ class GitLabPromptManager(CustomPromptManagement):
if parsed_messages:
final_messages: list[AllMessageValues] = parsed_messages
else:
final_messages = [{"role": "user", "content": rendered_prompt}] + messages # type: ignore
final_messages = [{"role": "user", "content": rendered_prompt}] + messages
if litellm_params is None:
litellm_params = {}
@ -400,7 +400,7 @@ class GitLabPromptManager(CustomPromptManagement):
"role": current_role,
"content": "\n".join(current_content).strip(),
}
) # type: ignore
)
current_role = "system"
current_content = [line[7:].strip()]
elif low.startswith("user:"):
@ -410,7 +410,7 @@ class GitLabPromptManager(CustomPromptManagement):
"role": current_role,
"content": "\n".join(current_content).strip(),
}
) # type: ignore
)
current_role = "user"
current_content = [line[5:].strip()]
elif low.startswith("assistant:"):
@ -420,16 +420,16 @@ class GitLabPromptManager(CustomPromptManagement):
"role": current_role,
"content": "\n".join(current_content).strip(),
}
) # type: ignore
)
current_role = "assistant"
current_content = [line[10:].strip()]
else:
current_content.append(line)
if current_role and current_content:
messages.append({"role": current_role, "content": "\n".join(current_content).strip()}) # type: ignore
messages.append({"role": current_role, "content": "\n".join(current_content).strip()})
if not messages and prompt_content.strip():
messages = [{"role": "user", "content": prompt_content.strip()}] # type: ignore
messages = [{"role": "user", "content": prompt_content.strip()}]
return messages
def post_call_hook(

6
litellm/integrations/lago.py
View file

@ -23,9 +23,9 @@ def get_utc_datetime():
from datetime import datetime
if hasattr(dt, "UTC"):
return datetime.now(dt.UTC) # type: ignore
return datetime.now(dt.UTC)
else:
return datetime.utcnow() # type: ignore
return datetime.utcnow()
class LagoLogger(CustomLogger):
@ -92,7 +92,7 @@ class LagoLogger(CustomLogger):
"user_id",
"team_id",
]:
charge_by = os.environ["LAGO_API_CHARGE_BY"] # type: ignore
charge_by = os.environ["LAGO_API_CHARGE_BY"]
else:
raise Exception("invalid LAGO_API_CHARGE_BY set")

14
litellm/integrations/langfuse/langfuse.py
View file

@ -433,14 +433,14 @@ class LangFuseLogger:
input,
response_obj,
):
from langfuse.model import CreateGeneration, CreateTrace # type: ignore
from langfuse.model import CreateGeneration, CreateTrace
verbose_logger.warning(
"Please upgrade langfuse to v2.0.0 or higher: https://github.com/langfuse/langfuse-python/releases/tag/v2.0.1"
)
trace: Final = self.Langfuse.trace( # type: ignore
CreateTrace( # type: ignore
trace: Final = self.Langfuse.trace(
CreateTrace(
name=metadata.get("generation_name", "litellm-completion"),
input=input,
output=output,
@ -959,8 +959,8 @@ class LangFuseLogger:
"guardrail_mode": guardrail_entry.get("guardrail_mode", None),
"guardrail_masked_entity_count": guardrail_entry.get("masked_entity_count", None),
},
start_time=guardrail_entry.get("start_time", None), # type: ignore
end_time=guardrail_entry.get("end_time", None), # type: ignore
start_time=guardrail_entry.get("start_time", None),
end_time=guardrail_entry.get("end_time", None),
)
verbose_logger.debug("Logged guardrail information as span: %s", span)
@ -1006,7 +1006,7 @@ def _add_prompt_to_generation_params(
if "labels" in prompt_text_params and "tags" in prompt_text_params:
_data["labels"] = user_prompt.get("labels", []) or []
_data["tags"] = user_prompt.get("tags", []) or []
_prompt_obj = Prompt_Text(**_data) # type: ignore
_prompt_obj = Prompt_Text(**_data)
generation_params["prompt"] = TextPromptClient(prompt=_prompt_obj)
elif isinstance(user_prompt["prompt"], list):
@ -1021,7 +1021,7 @@ def _add_prompt_to_generation_params(
_data["labels"] = user_prompt.get("labels", []) or []
_data["tags"] = user_prompt.get("tags", []) or []
_prompt_obj = Prompt_Chat(**_data) # type: ignore
_prompt_obj = Prompt_Chat(**_data)
generation_params["prompt"] = ChatPromptClient(prompt=_prompt_obj)
else:

6
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -2,7 +2,7 @@ import base64
import json
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Optional, Union
from typing import TYPE_CHECKING, Any, Final, Optional
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
@ -18,7 +18,7 @@ from litellm.types.utils import StandardCallbackDynamicParams
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = Union[_Span, Any]
Span = _Span | Any
else:
Span = Any
@ -74,7 +74,7 @@ class LangfuseOtelLogger(OpenTelemetry):
LangFuseLogger as _LFLogger,
)
metadata = _LFLogger.add_metadata_from_header(litellm_params, metadata) # type: ignore
metadata = _LFLogger.add_metadata_from_header(litellm_params, metadata)
except Exception:
# Fallback silently if import fails; header enrichment just won't happen
pass

Some files were not shown because too many files have changed in this diff Show more