Merge branch 'main' into litellm_fix_passthrough_failure_model_from_url

This commit is contained in:
mateo-berri 2026-10-28 17:27:42 -07:00
commit 80dfcc1eee
203 changed files with 6614 additions and 922 deletions

View file

@ -561,6 +561,9 @@ jobs:
- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
- name: router_surface_check
run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
- name: test_chat_completion_imports
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py

View file

@ -245,6 +245,7 @@ jobs:
tests/unit/llms/clarifai
tests/unit/llms/claude_code
tests/unit/llms/cloudflare
tests/unit/llms/codestral
tests/unit/llms/codex
tests/unit/llms/cohere
tests/unit/llms/cometapi

View file

@ -2,11 +2,12 @@
needs an approving review from the spend-log owner before it can merge.
CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
so a path rule would put every migration on the owner. This gate reads the diff instead: when no
added line in the watched files mentions the table it passes at once, otherwise it passes only once
the owner's latest review on the pull request is an approval that still covers every spend-log line
in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never
saw). Reviews do not trigger the workflow, so the check is re-run by hand after approving.
so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
only once the owner's latest review on the pull request is an approval that still covers every spend-log line
in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
Reviews do not trigger the workflow, so the check is re-run by hand after approving.
"""
from __future__ import annotations
@ -15,8 +16,10 @@ import argparse
import re
import subprocess
import sys
from collections.abc import Callable, Sequence
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, TypeAdapter
@ -29,6 +32,8 @@ WATCHED: Final = (
MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
class Reviewer(BaseModel):
@ -46,26 +51,71 @@ class Review(BaseModel):
_REVIEWS: Final = TypeAdapter(list[Review])
@dataclass(frozen=True, slots=True)
class Block:
"""Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
base: range
head: range
@dataclass(frozen=True, slots=True)
class Hunk:
removed: tuple[tuple[int, str], ...]
added: tuple[tuple[int, str], ...]
def watched(path: str) -> bool:
return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
def _added_lines(section: str) -> tuple[str, ...]:
body: Final = section.split("\n@@", 1)
return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
def model_block(schema: str) -> range:
"""1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
lines: Final = schema.splitlines()
starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
if not starts:
return range(0)
ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
return range(starts[0], (ends[0] if ends else len(lines)) + 1)
def spend_log_hits(diff: str) -> tuple[str, ...]:
"""`path: added line` for every added line in a watched file that names the table."""
def _hunks(section: str) -> tuple[Hunk, ...]:
headers: Final = tuple(HUNK.finditer(section))
ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
def _hunk(header: re.Match[str], body: str) -> Hunk:
lines: Final = body.splitlines()
return Hunk(
removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
)
def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
"""`path: +line` or `path: -line` for every changed line that touches the table.
A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
merge base for removed ones), a migration line when it is added and names the table."""
sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
pairs: Final = tuple((_path_of(section), section) for section in sections)
paths: Final = tuple((_path_of(section), section) for section in sections)
return tuple(
(path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
) # comprehension-ok: path then its lines
hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
) # comprehension-ok: path then its hits
def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
if path not in blocks:
return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
block: Final = blocks[path]
removed: Final = tuple(
f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
) # comprehension-ok: hunk then its lines
added: Final = tuple(
f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
) # comprehension-ok: hunk then its lines
return removed + added
def _path_of(section: str) -> str:
@ -111,13 +161,25 @@ def added_diff(base: str, head: str) -> str:
return result.stdout
def file_at(revision: str, path: str) -> str:
"""Contents of `path` at `revision`, empty when the file does not exist there."""
result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
return result.stdout if result.returncode == 0 else ""
def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
return MappingProxyType(
{path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
)
def fetched(commit: str) -> bool:
result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
return result.returncode == 0
def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
return spend_log_hits(added_diff(base, commit)) if fetched(commit) else None
return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
class Arguments(BaseModel):
@ -135,9 +197,12 @@ def main(argv: Sequence[str]) -> int:
parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
hits: Final = spend_log_hits(added_diff(args.base, args.head))
hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
if not hits:
sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
sys.stdout.write(
"No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
"nothing to review.\n"
)
return 0
reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None

View file

@ -4393,6 +4393,10 @@ dependencies = [
"rstest",
]
[[package]]
name = "litellm-router-core"
version = "0.1.0"
[[package]]
name = "litellm-router-types"
version = "0.1.0"

View file

@ -1,3 +1,36 @@
//! The model catalog: per-model pricing, context limits and capabilities.
//!
//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
//! in `litellm-router-core`, which reads this catalog through its dynamic config.
//!
//! # Structure
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
//! and keeps every entry in file order with its raw fields.
//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
//! modalities and modes in `capabilities`).
//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
//! failing the parse.
//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
//! families. They are data in the file, not entries.
//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
//! lost most of its models is refused), and `validate_registry` checks every entry against the
//! checked-in contract.
//!
//! # Lookups
//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
//! prefixes are not stripped here; callers decide which keys to try.
//!
//! # Readers
//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
//! performs I/O; loading and refreshing the file is the caller's job.
mod capabilities;
mod catalog;
mod error;

View file

@ -0,0 +1,9 @@
[package]
name = "litellm-router-core"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
[dependencies]

View file

@ -0,0 +1,63 @@
//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks
//! and cooldowns around the attempts.
//!
//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in
//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the
//! same interface. Until then it holds this overview and nothing else, so the interface is written
//! down in code before either side changes.
//!
//! # The public surface
//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else:
//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the
//! files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over
//! `call(op, request)`.
//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`,
//! `observe()`, `register_plugin()` and `retire()`.
//!
//! Every other member that outside code uses today has a named replacement among those. The old
//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which
//! implements them over the entry points and warns on use. A CI check
//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside
//! code touches; the list may only shrink.
//!
//! # Structure
//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern
//! and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`,
//! `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and
//! `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing
//! and capabilities are the model catalog (`litellm-model-catalog`), which the router reads
//! through its dynamic config.
//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The
//! router never reads process globals. Static config (model list, strategy, retry, timeout and
//! cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config
//! (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read
//! once per call.
//! - `Settings`: one typed, frozen view of the configuration a caller may read.
//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and
//! Rust workers see the same state. `report(id, outcome)` records outcomes the router did not
//! see itself (a health check, an interceptor that marks a deployment failed).
//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the
//! configured strategy, without calling it.
//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing
//! a new registry version.
//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
//! policy is exhausted.
//!
//! # Execution
//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
//! started attempt is never replayed on a different executor.
//!
//! # Hosts
//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
//! serves them in the Rust server, where an operation without a Rust implementation makes the
//! config fail at build time, never partway through a request.
//!
//! # Plugins
//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
//! the current request), never the router itself.

View file

@ -19,6 +19,7 @@ from litellm.a2a_protocol.utils import (
)
from litellm.exceptions import BadRequestError
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33
RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256
@ -50,7 +51,7 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = (
class _SSELineSource(Protocol):
"""Minimal streaming-response surface used to read SSE lines."""
def aiter_lines(self) -> AsyncIterator[str]: ...
def aiter_text(self) -> AsyncIterator[str]: ...
def _filter_reserved_headers(
@ -244,7 +245,7 @@ class BedrockAgentCoreA2ATransformation:
Yields:
Parsed JSON dicts from SSE data lines
"""
async for line in response.aiter_lines():
async for line in aiter_sse_lines(response.aiter_text()):
line = line.strip()
if not line:
continue

View file

@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
from litellm.types.llms.custom_http import httpxSpecialProvider
_IBM_CLOUD_IAM_URL: Final = "https://iam.cloud.ibm.com/identity/token"
@ -74,7 +75,7 @@ class _WXORun(TypedDict, total=False):
class _SSELineSource(Protocol):
def aiter_lines(self) -> AsyncIterator[str]: ...
def aiter_text(self) -> AsyncIterator[str]: ...
class _WXOView(TypedDict, total=False):
@ -231,7 +232,7 @@ class WatsonxOrchestrateHandler:
async def _accumulate_wxo_sse_text(response: _SSELineSource) -> str:
source: Final[_WXOView] = {"sse_source": response}
accumulated_text = ""
async for line in source["sse_source"].aiter_lines():
async for line in aiter_sse_lines(source["sse_source"].aiter_text()):
if not line.startswith("data:"):
continue
data_str = line[5:].strip()

View file

@ -274,7 +274,9 @@ class InMemoryCache(BaseCache):
) -> list[float] | None:
results: Final = []
for increment in increment_list:
result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs)
result = await self.async_increment(
increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs
)
results.append(result)
return results

View file

@ -34,6 +34,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
)
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool
from litellm.responses.sse_output_recovery import (
parse_sse_json_chunk,
@ -1351,7 +1352,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
recovered_output_items: Final[dict[int, dict[str, object]]] = {}
recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
for chunk in raw_sse.splitlines():
for chunk in split_sse_lines(raw_sse):
parsed_chunk = parse_sse_json_chunk(chunk)
if parsed_chunk is None:
continue

View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
@ -131,9 +132,9 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
self.model = model
self.generate_content_provider_config = generate_content_provider_config
self.litellm_metadata = litellm_metadata
# Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
# Gemini streamGenerateContent uses SSE line framing; iter_sse_lines keeps
# large inlineData payloads (e.g. image/jpeg) intact within one event.
self.stream_iterator = response.iter_lines()
self.stream_iterator = iter_sse_lines(response.iter_text())
def __iter__(self):
return self
@ -182,9 +183,9 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
self.model = model
self.generate_content_provider_config = generate_content_provider_config
self.litellm_metadata = litellm_metadata
# Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
# Gemini streamGenerateContent uses SSE line framing; aiter_sse_lines keeps
# large inlineData payloads (e.g. image/jpeg) intact within one event.
self.stream_iterator = response.aiter_lines()
self.stream_iterator = aiter_sse_lines(response.aiter_text())
def __aiter__(self):
return self

View file

@ -1,9 +1,9 @@
import asyncio
import contextvars
import importlib
from collections.abc import Coroutine, Mapping
from collections.abc import Collection, Coroutine, Mapping
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, cast, overload
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
from pydantic import TypeAdapter, ValidationError
@ -61,6 +61,7 @@ from litellm.types.utils import (
ProviderRoutingHints,
)
from litellm.utils import (
EXTRA_BODY_ROUTING_KEYS,
ImageResponse,
ProviderConfigManager,
get_llm_provider,
@ -466,7 +467,7 @@ def image_generation(
litellm.LlmProviders.QWENCLOUD,
litellm.LlmProviders.QWEN_AI_PLATFORM,
litellm.LlmProviders.EDENAI,
):
) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
@ -807,6 +808,32 @@ def image_variation(
return response
_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams))
_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"})
_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...])
def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
try:
return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params)
except ValidationError:
return ()
def _tokenhub_passthrough_edit_params(
non_default_params: Mapping[str, object],
extra_body: Mapping[str, object] | None,
mapped_params: Collection[str],
additional_drop_params: Collection[str] | None,
) -> Mapping[str, object]:
dropped: Final = frozenset(additional_drop_params or ())
unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped
return {
**{key: value for key, value in non_default_params.items() if key not in unmapped_excluded},
**{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS},
}
@client
def image_edit(
image: FileTypes | list[FileTypes] | None = None,
@ -935,22 +962,24 @@ def image_edit(
raise ValueError(f"image edit is not supported for {custom_llm_provider}")
local_vars.update(kwargs)
provider_supported_params: Final[frozenset[str]] = frozenset(
image_edit_provider_config.get_supported_openai_params(model)
)
# Get ImageEditOptionalRequestParams with only valid parameters
image_edit_optional_params: Final[ImageEditOptionalRequestParams] = (
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
local_vars,
provider_supported_params=frozenset(
image_edit_provider_config.get_supported_openai_params(model)
).intersection(non_default_params),
provider_supported_params=provider_supported_params.intersection(non_default_params),
)
)
# Get optional parameters for the responses API
additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params"))
image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
drop_params=kwargs.get("drop_params"),
additional_drop_params=kwargs.get("additional_drop_params"),
additional_drop_params=additional_drop_params,
)
if image_edit_provider_config.use_multipart_form_data() and (
@ -1039,7 +1068,19 @@ def image_edit(
image=images,
prompt=prompt,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_request_params=image_edit_request_params,
image_edit_optional_request_params={
**image_edit_request_params,
**(
_tokenhub_passthrough_edit_params(
non_default_params,
extra_body if isinstance(extra_body, dict) else None,
provider_supported_params,
additional_drop_params,
)
if custom_llm_provider == litellm.LlmProviders.TENCENT.value
else {}
),
},
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,

View file

@ -1,6 +1,9 @@
import base64
import os
from collections.abc import Collection, Mapping
from io import BufferedReader, BytesIO
from typing import Any, Final, cast, get_type_hints
from pathlib import Path
from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
import litellm
from litellm.litellm_core_utils.token_counter import get_image_type
@ -16,7 +19,7 @@ class ImageEditRequestUtils:
image_edit_provider_config: BaseImageEditConfig,
image_edit_optional_params: ImageEditOptionalRequestParams,
drop_params: bool | None = None,
additional_drop_params: list[str] | None = None,
additional_drop_params: Collection[str] | None = None,
) -> dict:
"""
Get optional parameters for the image edit API.
@ -145,3 +148,36 @@ class ImageEditRequestUtils:
except Exception:
# If anything goes wrong, default to PNG
return FILE_MIME_TYPES[FileType.PNG]
@runtime_checkable
class _SeekableBinaryReader(Protocol):
def tell(self) -> int: ...
def seek(self, offset: int) -> int: ...
def read(self) -> bytes: ...
def _read_image_bytes(image: object) -> bytes:
if isinstance(image, bytes):
return image
if isinstance(image, tuple):
return _read_image_bytes(image[1])
if isinstance(image, os.PathLike):
return Path(image).read_bytes()
if isinstance(image, _SeekableBinaryReader):
position: Final = image.tell()
image.seek(0)
data: Final = image.read()
image.seek(position)
return data
raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}")
def image_to_data_url(image: object) -> str:
if isinstance(image, str):
return image
image_bytes: Final = _read_image_bytes(image)
mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"

View file

@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
_RESPONSE_CONTENT_FIELD: Final = "content"
_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
_ResponseT: Final = TypeVar("_ResponseT")
@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger):
get_last_user_message,
)
query: Final = get_last_user_message(cast(list[AllMessageValues], messages))
last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or ""
query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip()
if not query:
return None

View file

@ -21,6 +21,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.types.interactions import (
InteractionsAPIStreamingResponse,
)
@ -145,7 +146,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
)
self.stream_iterator = response.aiter_lines()
self.stream_iterator = aiter_sse_lines(response.aiter_text())
def __aiter__(self):
return self
@ -212,7 +213,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
)
self.stream_iterator = response.iter_lines()
self.stream_iterator = iter_sse_lines(response.iter_text())
def __iter__(self):
return self

View file

@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
"rpm",
"itpm",
"otpm",
"tpd",
"rpd",
"use_xai_oauth",
"chatgpt_auth_file",
"github_copilot_auth_type",

View file

@ -3481,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
level=LogfireLevel.INFO.value,
level=LogfireLevel.INFO,
)
if callback == "lunary" and lunaryLogger is not None:
@ -4442,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass):
response_obj=result,
start_time=start_time,
end_time=end_time,
level=LogfireLevel.ERROR.value,
level=LogfireLevel.ERROR,
print_verbose=print_verbose,
)

View file

@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone, tzinfo
from types import MappingProxyType
from typing import Final, Literal, TypedDict, cast
from typing import Final, Literal, Protocol, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost(
)
class _ModelPricedImageCostCalculator(Protocol):
def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ...
def _model_priced_image_cost_calculator(
custom_llm_provider: str | None, model: str
) -> _ModelPricedImageCostCalculator | None:
match custom_llm_provider:
case litellm.LlmProviders.RECRAFT.value:
from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator
return recraft_image_cost_calculator
case litellm.LlmProviders.AIML.value:
from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator
return aiml_image_cost_calculator
case litellm.LlmProviders.COMETAPI.value:
from litellm.llms.cometapi.image_generation.cost_calculator import (
cost_calculator as cometapi_image_cost_calculator,
)
return cometapi_image_cost_calculator
case litellm.LlmProviders.RUNWAYML.value:
from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator
return runwayml_image_cost_calculator
case litellm.LlmProviders.TENCENT.value:
from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
case _:
return None
class CostCalculatorUtils:
@staticmethod
def call_type_has_image_response(call_type: str) -> bool:
@ -2131,9 +2166,6 @@ class CostCalculatorUtils:
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
cost_calculator as vertex_ai_image_cost_calculator,
)
@ -2146,7 +2178,10 @@ class CostCalculatorUtils:
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
pricing: Final = deployment_pricing(model_info)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
if model_priced_calculator is not None:
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
if isinstance(completion_response, ImageResponse):
return vertex_ai_image_cost_calculator(
@ -2164,36 +2199,6 @@ class CostCalculatorUtils:
optional_params=optional_params,
)
raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation")
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
return recraft_image_cost_calculator(
model=model,
image_response=completion_response,
model_info=pricing,
)
elif custom_llm_provider == litellm.LlmProviders.AIML.value:
from litellm.llms.aiml.image_generation.cost_calculator import (
cost_calculator as aiml_image_cost_calculator,
)
return aiml_image_cost_calculator(
model=model,
image_response=completion_response,
model_info=pricing,
)
elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
from litellm.llms.cometapi.image_generation.cost_calculator import (
cost_calculator as cometapi_image_cost_calculator,
)
return cometapi_image_cost_calculator(
model=model,
image_response=completion_response,
model_info=pricing,
)
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
if call_type in (
CallTypes.image_edit.value,
@ -2237,16 +2242,6 @@ class CostCalculatorUtils:
optional_params=optional_params,
model_info=pricing,
)
elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
from litellm.llms.runwayml.cost_calculator import (
cost_calculator as runwayml_image_cost_calculator,
)
return runwayml_image_cost_calculator(
model=model,
image_response=completion_response,
model_info=pricing,
)
elif (
custom_llm_provider == litellm.LlmProviders.OPENAI.value
or custom_llm_provider == litellm.LlmProviders.AZURE.value

View file

@ -21,8 +21,8 @@ from pydantic import TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.batch_utils import InMemoryFile
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,

View file

@ -34,7 +34,7 @@ def safe_json_structure(
def _transform(key: str | None, value: str) -> str:
return value if value_transform is None else value_transform(key, value)
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
# Check for maximum depth.
if depth > max_depth:
return "MaxDepthExceeded"
@ -48,32 +48,32 @@ def safe_json_structure(
if id(obj) in seen:
return "CircularReference Detected"
seen.add(id(obj))
result: dict | list | tuple | set | str
if isinstance(obj, dict):
result = {}
for k, v in obj.items():
if isinstance(k, (str)):
clean_k = k.replace("\x00", "") if "\x00" in k else k
result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
mapping_result: Final = {
(clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
for k, v in obj.items()
if isinstance(k, str)
}
seen.remove(id(obj))
return result
return mapping_result
elif isinstance(obj, list):
result = [_serialize(item, seen, depth + 1, key) for item in obj]
list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
seen.remove(id(obj))
return result
return list_result
elif isinstance(obj, tuple):
result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
seen.remove(id(obj))
return result
return tuple_result
elif isinstance(obj, set):
result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
set_result: Final = sorted(sortable_values)
seen.remove(id(obj))
return result
return set_result
elif isinstance(obj, BaseModel):
dumped: Final = obj.model_dump()
result = _serialize(dumped, seen, depth + 1, key)
model_result: Final = _serialize(dumped, seen, depth + 1, key)
seen.remove(id(obj))
return result
return model_result
else:
# Fall back to string conversion for non-serializable objects.
try:

View file

@ -1263,7 +1263,7 @@ class CustomStreamWrapper:
)
if _is_delta_empty:
model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned
model_response.choices[0].delta = Delta(content=self._close_open_think_block())
# get any function call arguments
model_response.choices[0].finish_reason = map_finish_reason(
finish_reason=self.received_finish_reason
@ -1292,21 +1292,26 @@ class CustomStreamWrapper:
if self.merge_reasoning_content_in_choices is not True:
return
delta: Final = model_response.choices[0].delta
reasoning_content: Final = getattr(delta, "reasoning_content", None)
reasoning_content: Final = getattr(delta, "reasoning_content", None) or ""
content: Final = delta.content or ""
if reasoning_content:
self._emitted_disqualifying_content = True
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
opening: Final = "" if self.sent_first_thinking_block else "<think>"
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
self.sent_first_thinking_block = True
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
delta.content = "</think>" + content
self.sent_last_thinking_block = True
ends_thinking: Final = bool(content or delta.tool_calls or delta.function_call)
inside_think_block: Final = self.sent_first_thinking_block and not self.sent_last_thinking_block
opening: Final = "<think>" if reasoning_content and not inside_think_block else ""
closing: Final = "</think>" if ends_thinking and (reasoning_content or inside_think_block) else ""
if hasattr(delta, "reasoning_content"):
del delta.reasoning_content
if not reasoning_content and not closing:
return
delta.content = opening + reasoning_content + closing + content
self.sent_first_thinking_block = True
self.sent_last_thinking_block = bool(closing)
self._emitted_disqualifying_content = self._emitted_disqualifying_content or bool(reasoning_content)
def _close_open_think_block(self) -> str | None:
if not self.sent_first_thinking_block or self.sent_last_thinking_block:
return None
self.sent_last_thinking_block = True
return "</think>"
def _dispatch_provider_chunk(
self,
@ -1932,8 +1937,20 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason == "stop" and self.tool_call
): # don't overwrite for other - potential error finish reasons
model_response.choices[0].finish_reason = "tool_calls"
close_tag: Final = self._close_open_think_block()
if close_tag is not None:
model_response.choices[0].delta.content = close_tag
return model_response
def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
return processed_chunk
if processed_chunk.choices[0].delta.content is None:
return None
return processed_chunk.model_copy(
update={"choices": [processed_chunk.choices[0].model_copy(update={"finish_reason": None})]}
)
def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
"""
Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
@ -2173,10 +2190,9 @@ class CustomStreamWrapper:
else:
self.sent_last_chunk = True
processed_chunk: Final = self.finish_reason_handler()
# The logged response is built from self.chunks; keep a finish_reason the provider sent on its
# last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
self.chunks.append(processed_chunk)
final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
if final_chunk_for_logging is not None:
self.chunks.append(final_chunk_for_logging)
if self.stream_options is None:
self._set_final_chunk_usage(processed_chunk)
## LOGGING
@ -2406,10 +2422,9 @@ class CustomStreamWrapper:
else:
self.sent_last_chunk = True
processed_chunk: Final = self.finish_reason_handler()
# The logged response is built from self.chunks; keep a finish_reason the provider sent on its
# last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
self.chunks.append(processed_chunk)
final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
if final_chunk_for_logging is not None:
self.chunks.append(final_chunk_for_logging)
if self.stream_options is None:
self._set_final_chunk_usage(processed_chunk)
# see sync __next__'s sibling branch: deliberately do NOT restore

View file

@ -38,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
get_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.types.llms.anthropic import (
ContentBlockDelta,
ContentBlockStart,
@ -233,7 +234,7 @@ async def make_call(
raise AnthropicError(status_code=500, message=str(e))
completion_stream: Final = ModelResponseIterator(
streaming_response=response.aiter_lines(),
streaming_response=aiter_sse_lines(response.aiter_text()),
sync_stream=False,
json_mode=json_mode,
speed=speed,
@ -304,7 +305,7 @@ def make_sync_call(
)
completion_stream: Final = ModelResponseIterator(
streaming_response=response.iter_lines(),
streaming_response=iter_sse_lines(response.iter_text()),
sync_stream=True,
json_mode=json_mode,
speed=speed,

View file

@ -13,6 +13,7 @@ from litellm.llms.anthropic.chat.handler import (
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.types.utils import (
Choices,
Message,
@ -269,7 +270,7 @@ def build_complete_streaming_response_legacy(
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
for line in event_str.splitlines():
for line in split_sse_lines(event_str):
stripped = line.strip()
if stripped.startswith("data:"):
payload = stripped[len("data:") :].strip()

View file

@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
from litellm.router_utils.baseline_request import (
from litellm.router_strategy.auto_router.baseline_request import (
BASELINE_PARAMETERS,
capture_baseline_parameters,
)

View file

@ -35,6 +35,7 @@ from litellm.llms.azure_ai.agents.transformation import (
AzureAIAgentsConfig,
AzureAIAgentsError,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionAnnotationURLCitation,
@ -632,7 +633,7 @@ class AzureAIAgentsHandler:
current_event = None
async for line in response.aiter_lines():
async for line in aiter_sse_lines(response.aiter_text()):
line = line.strip()
if line.startswith("event:"):

View file

@ -96,7 +96,7 @@ class BaseModelResponseIterator:
released (and a backend like vLLM aborts generation) when the stream
is abandoned before its natural end.
``streaming_response`` is usually a bare ``aiter_lines()`` generator
``streaming_response`` is usually a bare ``aiter_sse_lines()`` generator
that holds no reference to the response, so the handler that owns the
response attaches it here after construction."""
if self.http_response is not None:

View file

@ -6,6 +6,7 @@ from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row ha
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@ -24,7 +25,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]:
events: Final[list[dict]] = []
for block in chunk.split("\n\n"):
for line in block.splitlines():
for line in split_sse_lines(block):
stripped = line.strip()
if not stripped.startswith("data:"):
continue

View file

@ -12,6 +12,7 @@ from typing import Final
import httpx
from pydantic import Field, PrivateAttr
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
from litellm.types.llms.base import LiteLLMPydanticObjectBase
SANDBOX_MAX_OUTPUT_BYTES: Final = 10 * 1024 * 1024
@ -99,7 +100,7 @@ class BaseSandboxConfig:
async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
lines: Final[list[str]] = []
total = 0
async for line in response.aiter_lines():
async for line in aiter_sse_lines(response.aiter_text()):
total += len(line.encode("utf-8"))
if total > SANDBOX_MAX_OUTPUT_BYTES:
raise ValueError(

View file

@ -10,6 +10,8 @@ import time
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -34,6 +36,15 @@ else:
LiteLLMLoggingObj = Any
@with_config(ConfigDict(extra="allow", strict=True))
class _PegasusResponse(TypedDict):
message: ReadOnly[NotRequired[str | None]]
finishReason: ReadOnly[NotRequired[str | None]]
_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
"""
Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
ModelResponse with choices[0].message.content and finish_reason
"""
try:
completion_response: Final = raw_response.json()
completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
except Exception as e:
raise BedrockError(
message=f"Error parsing response: {raw_response.text}, error: {e}",
@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
message_content: Final = completion_response.get("message", "")
# Extract finish reason and map to LiteLLM format
finish_reason_raw: Final = completion_response.get("finishReason", "stop")
finish_reason_raw: Final = completion_response.get("finishReason") or "stop"
finish_reason: Final = map_finish_reason(finish_reason_raw)
# Set the response content

View file

@ -1,11 +1,12 @@
import copy
import json
import time
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
import httpx
from pydantic import TypeAdapter, ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
_GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
@with_config(ConfigDict(extra="allow", strict=True))
class _CohereGeneration(TypedDict):
text: ReadOnly[str | None]
finish_reason: ReadOnly[str | None]
@with_config(ConfigDict(extra="allow", strict=True))
class _CohereInvokeResponse(TypedDict):
text: NotRequired[ReadOnly[str | None]]
generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
@with_config(ConfigDict(extra="allow", strict=True))
class _LlamaInvokeResponse(TypedDict):
generation: ReadOnly[str | None]
_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
_GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
outputText: str | None = None
try:
if provider == "cohere":
if "text" in completion_response:
outputText = completion_response["text"]
elif "generations" in completion_response:
outputText = completion_response["generations"][0]["text"]
cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
if "text" in cohere_response:
outputText = cohere_response["text"]
elif "generations" in cohere_response:
outputText = cohere_response["generations"][0]["text"]
model_response.choices[0].finish_reason = map_finish_reason(
completion_response["generations"][0]["finish_reason"]
cohere_response["generations"][0]["finish_reason"] or "stop"
)
elif provider == "anthropic":
return litellm.AmazonAnthropicClaudeConfig().transform_response(
@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
elif provider == "ai21":
outputText = completion_response.get("completions")[0].get("data").get("text")
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
outputText = completion_response["generation"]
outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
elif provider == "mistral":
outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
else: # amazon titan

View file

@ -49,6 +49,7 @@ from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
LiteLLMLoggingObj,
)
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.bedrock import AwsAuthParams, BedrockBatchRecordKind
from litellm.types.llms.openai import (
@ -400,7 +401,7 @@ def _batch_input_line(line_number: int, line: str) -> _BatchInputLine:
def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]:
numbered_lines: Final = enumerate(file_content.splitlines(), start=1)
numbered_lines: Final = enumerate(split_sse_lines(file_content), start=1)
return ((line_number, line) for line_number, line in numbered_lines if line.strip())

View file

@ -45,6 +45,7 @@ from litellm.llms.bedrock.passthrough.guardrail_translation.structured_edits imp
structured_edits,
tool_edits,
)
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.proxy.guardrails.auto_router_compression import is_compression_guardrail
from litellm.types.utils import GenericGuardrailAPIInputs
@ -947,7 +948,7 @@ def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]:
def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]:
return tuple(
_JSON_OBJECT.validate_python(json.loads(line[len("data:") :]))
for line in sse.decode().splitlines()
for line in split_sse_lines(sse.decode())
if line.startswith("data:")
)

View file

@ -53,6 +53,7 @@ from litellm.types.realtime import (
RealtimeResponseTypedDict,
)
_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
_JSON_VALUE: Final = TypeAdapter(object)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
try:
json_message: Final[dict[str, JsonValue]] = json.loads(message)
json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
except json.JSONDecodeError:
verbose_logger.warning("Invalid JSON message: %s", message[:200])
return []

View file

@ -47,6 +47,7 @@ from litellm.llms.base_llm.search.transformation import (
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.secret_managers.main import get_secret_str
# AgentCore web-search rejects queries longer than 200 characters
@ -146,7 +147,9 @@ def _iter_sse_events(text: str) -> Iterator[Mapping[str, object]]:
progress notifications before the JSON-RPC response.
"""
for chunk in _SSE_EVENT_SEPARATOR.split(text):
payload = "\n".join(line[len("data:") :].lstrip() for line in chunk.splitlines() if line.startswith("data:"))
payload = "\n".join(
line[len("data:") :].lstrip() for line in split_sse_lines(chunk) if line.startswith("data:")
)
if not payload:
continue
try:

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
safe_convert_created_field,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.sse_output_recovery import (
@ -199,7 +200,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
error_message = None
streamed_output_items: Final[dict[int, dict[str, object]]] = {}
text_only_output_items: Final[dict[int, dict[str, object]]] = {}
for chunk in body_text.splitlines():
for chunk in split_sse_lines(body_text):
parsed_chunk = parse_sse_json_chunk(chunk)
if parsed_chunk is None:
continue

View file

@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.types.utils import TextChoices
from litellm.utils import CustomStreamWrapper, TextCompletionResponse
@ -109,7 +110,7 @@ async def make_call(
if response.status_code != 200:
raise TextCompletionCodestralError(status_code=response.status_code, message=response.text)
completion_stream: Final = response.aiter_lines()
completion_stream: Final = aiter_sse_lines(response.aiter_text())
# LOGGING
logging_obj.post_call(
input=messages,
@ -339,13 +340,13 @@ class CodestralTextCompletion:
response = litellm.module_level_client.post(
completion_url,
headers=headers,
data=json.dumps(data),
data=json.dumps({**data, "stream": True}),
stream=stream,
)
_response: Final = CustomStreamWrapper(
response.iter_lines(),
iter_sse_lines(response.iter_text()),
model,
custom_llm_provider="codestral",
custom_llm_provider="text-completion-codestral",
logging_obj=logging_obj,
)
return _response

View file

@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_httpx_client,
get_ssl_configuration,
)
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
from litellm.types.llms.openai import FileTypes
from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
@ -466,7 +467,7 @@ class BaseLLMAIOHTTPHandler:
)
else:
completion_stream = provider_config.get_model_response_iterator(
streaming_response=response.iter_lines(), sync_stream=True
streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
)
# LOGGING

View file

@ -118,6 +118,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_httpx_client,
header_value,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
MockResponsesAPIStreamingIterator,
@ -1086,7 +1087,7 @@ class BaseLLMHTTPHandler:
completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
else:
completion_stream = provider_config.get_model_response_iterator_for_request(
streaming_response=response.iter_lines(),
streaming_response=iter_sse_lines(response.iter_text()),
sync_stream=True,
json_mode=json_mode,
optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),
@ -1229,7 +1230,7 @@ class BaseLLMHTTPHandler:
completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
else:
completion_stream = provider_config.get_model_response_iterator_for_request(
streaming_response=response.aiter_lines(),
streaming_response=aiter_sse_lines(response.aiter_text()),
sync_stream=False,
json_mode=json_mode,
optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),

View file

@ -0,0 +1,74 @@
"""
Split a decoded Server-Sent Events body into lines the way the SSE spec does.
httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028,
U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a
`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR
(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream)
"""
import io
import re
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
from typing import Final
_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
_SSE_LINE_WITH_BREAK: Final = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+")
def _split_lines(text: str) -> Sequence[str]:
if "\r" not in text:
return text.split("\n")
crlf_folded: Final = text.replace("\r\n", "\n")
if "\r" not in crlf_folded:
return crlf_folded.split("\n")
return _SSE_LINE_BREAK.split(text)
class _SSELineSplitter:
def __init__(self) -> None:
self._partial_line: io.StringIO = io.StringIO()
self._after_cr: bool = False
def feed(self, text: str) -> tuple[str, ...]:
if not text:
return ()
fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text
self._after_cr = fresh.endswith("\r")
*ended, tail = _split_lines(fresh)
if not ended:
_ = self._partial_line.write(tail)
return ()
first: Final = self._partial_line.getvalue() + ended[0]
self._partial_line = io.StringIO()
_ = self._partial_line.write(tail)
return (first, *ended[1:])
def flush(self) -> tuple[str, ...]:
rest: Final = self._partial_line.getvalue()
self._partial_line = io.StringIO()
return (rest,) if rest else ()
def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]:
splitter: Final = _SSELineSplitter()
for text in text_chunks:
yield from splitter.feed(text)
yield from splitter.flush()
def split_sse_lines(text: str) -> tuple[str, ...]:
return tuple(iter_sse_lines((text,)))
def split_sse_lines_keepends(text: str) -> tuple[str, ...]:
return tuple(_SSE_LINE_WITH_BREAK.findall(text))
async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]:
splitter: Final = _SSELineSplitter()
async for text in text_chunks:
for line in splitter.feed(text):
yield line
for line in splitter.flush():
yield line

View file

@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
@with_config(ConfigDict(extra="allow", strict=True))
class _DashScopeUsage(TypedDict, total=False):
prompt_tokens: NotRequired[ReadOnly[int]]
total_tokens: NotRequired[ReadOnly[int]]
@with_config(ConfigDict(extra="allow", strict=True))
class _DashScopeEmbeddingResponse(TypedDict, total=False):
data: NotRequired[ReadOnly[list[object]]]
model: NotRequired[ReadOnly[str | None]]
usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
id: NotRequired[ReadOnly[object]]
_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
"""
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
message=message,
)
parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
model_response.object = "list"
model_response.data = response_json.get("data", [])
model_response.model = response_json.get("model", model)
model_response.data = parsed_response.get("data", [])
model_response.model = parsed_response.get("model", model)
usage: Final = response_json.get("usage") or {}
usage: Final = parsed_response.get("usage") or {}
prompt_tokens: Final = usage.get("prompt_tokens", 0)
total_tokens: Final = usage.get("total_tokens", prompt_tokens)
setattr(
@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
),
)
if "id" in response_json:
setattr(model_response, "id", response_json["id"])
if "id" in parsed_response:
setattr(model_response, "id", parsed_response["id"])
return model_response

View file

@ -1,6 +1,8 @@
import json
from typing import Final
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.types.llms.openai import (
@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
)
_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class ModelResponseIterator:
def __init__(self, streaming_response, sync_stream: bool):
self.streaming_response = streaming_response
@ -99,7 +104,7 @@ class ModelResponseIterator:
self.response_iterator = self.streaming_response
chunk: Final = self._next_sse_data()
try:
json_chunk: Final = json.loads(chunk)
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
return self.chunk_parser(chunk=json_chunk)
except ValueError as e:
verbose_logger.debug(
@ -126,7 +131,7 @@ class ModelResponseIterator:
if chunk == "[DONE]":
raise StopAsyncIteration
try:
json_chunk: Final = json.loads(chunk)
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
return self.chunk_parser(chunk=json_chunk)
except ValueError as e:
verbose_logger.debug(

View file

@ -4,13 +4,14 @@ from typing import Final
from httpx._types import RequestFiles
from litellm.images.utils import image_to_data_url
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes
from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url
from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig
FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth"
SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size")
@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
)
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
"prompt": prompt,
"image_url": to_data_url(next(iter(images))),
"image_url": image_to_data_url(next(iter(images))),
**provider_params,
}
return request_body, ()

View file

@ -1,15 +1,12 @@
import base64
import os
from collections.abc import Mapping
from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
from typing import TYPE_CHECKING, Final
import httpx
from httpx._types import RequestFiles
from pydantic import ConfigDict, TypeAdapter
from litellm.images.utils import ImageEditRequestUtils
from litellm.images.utils import image_to_data_url
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import (
map_gpt_image_quality,
@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
)
@runtime_checkable
class _SeekableBinaryReader(Protocol):
def tell(self) -> int: ...
def seek(self, offset: int) -> int: ...
def read(self) -> bytes: ...
def _read_image_bytes(image: object) -> bytes:
if isinstance(image, bytes):
return image
if isinstance(image, tuple):
return _read_image_bytes(image[1])
if isinstance(image, os.PathLike):
return Path(image).read_bytes()
if isinstance(image, _SeekableBinaryReader):
position: Final = image.tell()
image.seek(0)
data: Final = image.read()
image.seek(position)
return data
raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}")
def to_data_url(image: object) -> str:
if isinstance(image, str):
return image
image_bytes: Final = _read_image_bytes(image)
mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
def _first(value: object) -> object:
return value[0] if isinstance(value, list) and value else value
@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig):
raise ValueError("Fal AI image edit requires at least one input image")
mask: Final = _first(image_edit_optional_request_params.get("mask"))
mask_field: Final[Mapping[str, str]] = (
MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({})
)
provider_params: Final[Mapping[str, object]] = MappingProxyType(
{key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
)
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
"prompt": prompt,
"image_urls": tuple(to_data_url(img) for img in images),
"image_urls": tuple(image_to_data_url(img) for img in images),
**mask_field,
**provider_params,
}

View file

@ -23,6 +23,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamTransformSink,
)
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
@ -121,7 +122,7 @@ def _parse_json_dict_or_none(payload: str) -> Mapping[str, object] | None:
def _sse_payload_texts(sse_text: str) -> tuple[str, ...]:
return tuple(
text
for line in sse_text.splitlines()
for line in split_sse_lines(sse_text)
if line.startswith("data:")
for payload in (line[len("data:") :].strip(),)
if payload and payload != "[DONE]"

View file

@ -12,6 +12,8 @@ from types import MappingProxyType
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.caching.caching import InMemoryCache
@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
@with_config(ConfigDict(extra="allow", strict=True))
class _GigaChatTokenResponse(TypedDict):
tok: NotRequired[ReadOnly[str | None]]
access_token: NotRequired[ReadOnly[str | None]]
exp: NotRequired[ReadOnly[int | float | str | None]]
expires_at: NotRequired[ReadOnly[int | float | str | None]]
_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
class GigaChatAuthError(BaseLLMException):
"""GigaChat authentication error."""
@ -230,7 +243,7 @@ async def _request_token_async(
def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
"""Parse OAuth token response."""
data: Final = response.json()
data: Final = _TOKEN_RESPONSE.validate_python(response.json())
# GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
access_token: Final = data.get("tok") or data.get("access_token")

View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
@ -357,7 +358,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
api_key: str | None = None,
) -> list[dict[str, str]]:
streamed_response: Final = CustomStreamWrapper(
completion_stream=response.iter_lines(),
completion_stream=iter_sse_lines(response.iter_text()),
model=model,
custom_llm_provider="huggingface",
logging_obj=logging_obj,

View file

@ -12,6 +12,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
_DECODED_JSON: Final = TypeAdapter(object)
@ -36,12 +37,12 @@ class LangGraphSSEStreamIterator:
def __iter__(self):
"""Initialize sync iteration."""
self.line_iterator = self.response.iter_lines()
self.line_iterator = iter_sse_lines(self.response.iter_text())
return self
def __aiter__(self):
"""Initialize async iteration."""
self.async_line_iterator = self.response.aiter_lines()
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
return self
def _parse_sse_line(self, line: str) -> ModelResponseStream | None:

View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
from litellm.utils import EmbeddingResponse, ModelResponse, Usage
from ..common_utils import OobaboogaError
@ -74,7 +75,7 @@ def completion(
stream=optional_params["stream"] if "stream" in optional_params else False,
)
if "stream" in optional_params and optional_params["stream"] is True:
return response.iter_lines()
return iter_sse_lines(response.iter_text())
else:
return oobabooga_config.transform_response(
model=model,

View file

@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
import ssl
from typing import Any, Final, cast
from typing import TYPE_CHECKING, Final, cast
from litellm._logging import redact_string, verbose_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
from ..common_utils import is_openai_backed_api_base
from ..openai import OpenAIChatCompletion
if TYPE_CHECKING:
from fastapi import WebSocket
class OpenAIRealtime(OpenAIChatCompletion):
"""
@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
async def async_realtime(
self,
model: str,
websocket: Any,
websocket: "WebSocket",
logging_obj: LiteLLMLogging,
api_base: str | None = None,
api_key: str | None = None,

View file

@ -16,6 +16,7 @@ from litellm import LlmProviders
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai.openai import OpenAIConfig
@ -65,7 +66,9 @@ async def make_call(
model_response: Final = _fake_streamed_model_response(response.json())
completion_stream = MockResponseIterator(model_response=model_response)
else:
completion_stream = ModelResponseIterator(streaming_response=response.aiter_lines(), sync_stream=False)
completion_stream = ModelResponseIterator(
streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False
)
# LOGGING
logging_obj.post_call(
input=messages,
@ -103,7 +106,9 @@ def make_sync_call(
model_response: Final = _fake_streamed_model_response(response.json())
completion_stream = MockResponseIterator(model_response=model_response)
else:
completion_stream = ModelResponseIterator(streaming_response=response.iter_lines(), sync_stream=True)
completion_stream = ModelResponseIterator(
streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
)
# LOGGING
logging_obj.post_call(

View file

@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
"""
import base64
import builtins
import struct
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, Literal
import httpx
from pydantic import ConfigDict, TypeAdapter
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
_INT: Final = TypeAdapter(int)
@with_config(ConfigDict(extra="allow", strict=True))
class _PerplexityEmbeddingResponse(TypedDict, total=False):
model: ReadOnly[str | None]
object: ReadOnly[Literal["list"]]
data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
usage: ReadOnly[builtins.object]
_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
class PerplexityEmbeddingError(BaseLLMException):
def __init__(
self,
@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
litellm_params: dict = {},
) -> EmbeddingResponse:
try:
raw_response_json: Final = raw_response.json()
raw_response_json: Final[object] = raw_response.json()
except Exception:
raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
model_response.model = raw_response_json.get("model", model)
model_response.object = raw_response_json.get("object", "list")
response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
raw_response_json
)
raw_data: Final = raw_response_json.get("data", [])
model_response.model = response_data.get("model", model)
model_response.object = response_data.get("object", "list")
raw_data: Final = response_data.get("data", [])
decoded_data: Final[list[dict[str, object]]] = []
for item in raw_data:
decoded_item = dict(_JSON_OBJECT.validate_python(item))
@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
decoded_data.append(decoded_item)
model_response.data = decoded_data
usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
usage: Final = Usage(
prompt_tokens=_INT.validate_python(
usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0

View file

@ -14,6 +14,7 @@ from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.utils import CustomStreamWrapper, ModelResponse
from ..common_utils import PredibaseError
@ -34,7 +35,7 @@ async def make_call(
if response.status_code != 200:
raise PredibaseError(status_code=response.status_code, message=response.text)
completion_stream: Final = response.aiter_lines()
completion_stream: Final = aiter_sse_lines(response.aiter_text())
# LOGGING
logging_obj.post_call(
input=messages,
@ -163,7 +164,7 @@ class PredibaseChatCompletion:
timeout=timeout,
)
_response: Final = CustomStreamWrapper(
response.iter_lines(),
iter_sse_lines(response.iter_text()),
model,
custom_llm_provider="predibase",
logging_obj=logging_obj,

View file

@ -1,6 +1,49 @@
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.utils import (
calculate_image_response_cost_from_usage,
flat_image_cost,
generic_cost_per_token,
resolve_image_model_info,
)
from litellm.types.utils import ImageResponse, ModelInfo, Usage
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")
def _deployment_image_token_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
if model_info is None or model_info.get("output_cost_per_image_token") is not None:
return model_info
output_cost_per_token: Final = model_info.get("output_cost_per_token")
if output_cost_per_token is None:
return model_info
priced: Final[ModelInfo] = {**model_info, "output_cost_per_image_token": output_cost_per_token}
return priced
def _deployment_prices_per_image_only(model_info: ModelInfo | None) -> bool:
return (
model_info is not None
and model_info.get("output_cost_per_image") is not None
and model_info.get("output_cost_per_image_token") is None
and model_info.get("output_cost_per_token") is None
)
def image_cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
resolved_model_info: Final = resolve_image_model_info(
model=model, custom_llm_provider="tencent", model_info=_deployment_image_token_pricing(model_info)
)
if _deployment_prices_per_image_only(model_info):
return flat_image_cost(resolved_model_info, image_response)
token_based_cost: Final = calculate_image_response_cost_from_usage(
model=model,
image_response=image_response,
custom_llm_provider="tencent",
model_info=resolved_model_info,
)
if token_based_cost is not None:
return token_based_cost
return flat_image_cost(resolved_model_info, image_response)

View file

@ -0,0 +1,147 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeAlias
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from typing_extensions import override
from litellm.images.utils import image_to_data_url
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.tencent.image_generation.transformation import (
TOKENHUB_IMAGE_PARAMS,
tokenhub_image_headers,
tokenhub_image_params,
tokenhub_image_response,
tokenhub_image_url,
)
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_IMAGES: Final = TypeAdapter(tuple[object, ...])
_FORM_BOOLEANS: Final = MappingProxyType({"true": True, "false": False})
_OPENAI_PARAMS: Final = frozenset(TOKENHUB_IMAGE_PARAMS)
_FormEntries: TypeAlias = tuple[tuple[tuple[str, ...], object], ...]
def _form_scalar(value: object) -> object:
match value:
case str() if value.lower() in _FORM_BOOLEANS:
return _FORM_BOOLEANS[value.lower()]
case str() if value.removeprefix("-").isdigit():
return int(value)
case _:
return value
def _form_path(key: str) -> tuple[str, ...]:
root, bracket, rest = key.partition("[")
if not bracket or not rest.endswith("]"):
return (key,)
return (root, *rest.removesuffix("]").split("]["))
def _nested_form_value(entries: _FormEntries) -> object:
match entries:
case (((), value),):
return _form_scalar(value)
case _:
return _nested_form_fields(tuple(entry for entry in entries if entry[0]))
def _nested_form_fields(entries: _FormEntries) -> Mapping[str, object]:
roots: Final = tuple(dict.fromkeys(path[0] for path, _ in entries))
return {
root: _nested_form_value(tuple((path[1:], value) for path, value in entries if path[0] == root))
for root in roots
}
def _tokenhub_form_params(params: Mapping[str, object]) -> Mapping[str, object]:
tokenhub_entries: Final = tuple(
(_form_path(key), value) for key, value in params.items() if key not in _OPENAI_PARAMS
)
return {
**{key: value for key, value in params.items() if key in _OPENAI_PARAMS},
**_nested_form_fields(tokenhub_entries),
}
class TencentImageEditConfig(BaseImageEditConfig):
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
return list(TOKENHUB_IMAGE_PARAMS)
@override
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: mirrors override contract
return dict(tokenhub_image_params(image_edit_optional_params, model, drop_params))
@override
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None = None,
litellm_params: Mapping[str, object] | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
return dict(tokenhub_image_headers(headers, api_key, model))
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: Mapping[str, object],
) -> str:
return tokenhub_image_url(api_base)
@override
def use_multipart_form_data(self) -> bool:
return False
@override
def transform_image_edit_request(
self,
model: str,
prompt: str | None,
image: object,
image_edit_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
headers: Mapping[str, str],
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
images: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
return {
"model": model,
"prompt": prompt,
"images": tuple(image_to_data_url(img) for img in images if img is not None),
**_tokenhub_form_params(image_edit_optional_request_params),
}, ()
@override
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> ImageResponse:
image_response: Final = tokenhub_image_response(raw_response)
if image_response is None:
raise BaseLLMException(
status_code=raw_response.status_code,
message=raw_response.text,
headers=raw_response.headers,
)
return image_response

View file

@ -0,0 +1,200 @@
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import override
from litellm.exceptions import AuthenticationError, UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import (
ImageObject,
ImageResponse,
ImageUsage,
ImageUsageInputTokensDetails,
LlmProviders,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
TOKENHUB_API_BASE: Final = "https://tokenhub-intl.tencentcloudmaas.com/v1"
TOKENHUB_IMAGE_PATH: Final = "/wand/si-image/generation"
TOKENHUB_IMAGE_MODEL_PREFIX: Final = "seedream-"
TOKENHUB_IMAGE_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
"n",
"size",
"response_format",
"output_format",
"background",
"user",
)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
class _TokenHubImage(BaseModel):
model_config = ConfigDict(frozen=True)
url: str | None = None
b64_json: str | None = None
size: str | None = None
output_format: str | None = None
class _TokenHubUsage(BaseModel):
model_config = ConfigDict(frozen=True)
total_tokens: int
class _TokenHubImageResponse(BaseModel):
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
created: int
data: tuple[_TokenHubImage, ...]
tokenhub_usage: _TokenHubUsage | None = None
def is_tokenhub_image_model(model: str) -> bool:
return model.removeprefix(f"{LlmProviders.TENCENT.value}/").startswith(TOKENHUB_IMAGE_MODEL_PREFIX)
def tokenhub_image_url(api_base: str | None) -> str:
base: Final = api_base or get_secret_str("TENCENT_API_BASE") or TOKENHUB_API_BASE
return join_api_base(ensure_api_route(base, "v1"), TOKENHUB_IMAGE_PATH)
def tokenhub_image_headers(headers: Mapping[str, str], api_key: str | None, model: str) -> Mapping[str, str]:
resolved_api_key: Final = api_key or get_secret_str("TENCENT_API_KEY")
if not resolved_api_key:
raise AuthenticationError(
message="TENCENT_API_KEY is not set. Set it in the environment or pass api_key.",
llm_provider="tencent",
model=model,
)
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
def tokenhub_image_params(params: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, object]:
n: Final = params.get("n")
if n is not None and n != 1 and not drop_params:
raise UnsupportedParamsError(
status_code=400,
message=f"Tencent TokenHub returns one image per request, so n={n} is not supported for {model}. "
+ "To send n=1 instead, set `litellm.drop_params = True`.",
)
return {
key: value
for key, value in params.items()
if key != "n" and value is not None and not (key == "background" and value == "auto")
}
def tokenhub_image_response(raw_response: httpx.Response) -> ImageResponse | None:
try:
parsed: Final = _TokenHubImageResponse.model_validate_json(raw_response.content)
except ValidationError:
return None
total_tokens: Final = parsed.tokenhub_usage.total_tokens if parsed.tokenhub_usage is not None else 0
first_image: Final = parsed.data[0] if parsed.data else _TokenHubImage()
return ImageResponse(
created=parsed.created,
data=[ImageObject(url=image.url, b64_json=image.b64_json) for image in parsed.data],
size=first_image.size,
output_format=first_image.output_format,
usage=ImageUsage(
input_tokens=0,
input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0),
output_tokens=total_tokens,
total_tokens=total_tokens,
),
)
class TencentImageGenerationConfig(BaseImageGenerationConfig):
@override
def get_supported_openai_params(
self, model: str
) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: mirrors override contract
return list(TOKENHUB_IMAGE_PARAMS)
@override
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: mirrors override contract
return {**optional_params, **tokenhub_image_params(non_default_params, model, drop_params)}
@override
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
return tokenhub_image_url(api_base)
@override
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
return dict(tokenhub_image_headers(headers, api_key, model))
@override
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
headers: Mapping[str, str],
) -> dict[str, object]: # mutable-ok: mirrors override contract
extra_body: Final = _JSON_OBJECT.validate_python(optional_params.get("extra_body") or {})
params: Final = {key: value for key, value in optional_params.items() if key != "extra_body"}
return {"model": model, "prompt": prompt, **params, **extra_body}
@override
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: "LiteLLMLoggingObj",
request_data: Mapping[str, object],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
encoding: object,
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:
image_response: Final = tokenhub_image_response(raw_response)
if image_response is None:
raise BaseLLMException(
status_code=raw_response.status_code,
message=raw_response.text,
headers=raw_response.headers,
)
return image_response

View file

@ -22,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.llms.vertex_ai.agent_engine.sse_iterator import (
VertexAgentEngineResponseIterator,
)
@ -357,7 +358,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase):
) -> VertexAgentEngineResponseIterator:
"""Return a streaming iterator for SSE responses."""
return VertexAgentEngineResponseIterator(
streaming_response=raw_response.iter_lines(),
streaming_response=iter_sse_lines(raw_response.iter_text()),
sync_stream=True,
)
@ -465,7 +466,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase):
# Create iterator for SSE stream (async)
completion_stream: Final = VertexAgentEngineResponseIterator(
streaming_response=response.aiter_lines(),
streaming_response=aiter_sse_lines(response.aiter_text()),
sync_stream=False,
)

View file

@ -41,6 +41,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
get_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
@ -3241,7 +3242,7 @@ async def make_call(
raise
completion_stream: Final = ModelResponseIterator(
streaming_response=response.aiter_lines(),
streaming_response=aiter_sse_lines(response.aiter_text()),
sync_stream=False,
logging_obj=logging_obj,
response_headers=response.headers,
@ -3312,7 +3313,7 @@ def make_sync_call(
raise
completion_stream: Final = ModelResponseIterator(
streaming_response=response.iter_lines(),
streaming_response=iter_sse_lines(response.iter_text()),
sync_stream=True,
logging_obj=logging_obj,
response_headers=response.headers,
@ -4190,7 +4191,7 @@ class ModelResponseIterator:
log_failed_hop(e)
return self._truncated_stream_end(token)
self._start_hop(hop_response, token)
self.streaming_response = hop_response.aiter_lines()
self.streaming_response = aiter_sse_lines(hop_response.aiter_text())
self.async_response_iterator = self.streaming_response
return None
@ -4204,7 +4205,7 @@ class ModelResponseIterator:
log_failed_hop(e)
return self._truncated_stream_end(token)
self._start_hop(hop_response, token)
self.streaming_response = hop_response.iter_lines()
self.streaming_response = iter_sse_lines(hop_response.iter_text())
self.response_iterator = self.streaming_response
return None

View file

@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI
"""
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Final, cast
import httpx
from pydantic import ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -32,6 +33,17 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class _PredictionsEnvelope(TypedDict):
predictions: NotRequired[ReadOnly[dict[str, object]]]
_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope)
_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter(
_PredictionsEnvelopeType,
config=ConfigDict(strict=True, extra="allow"),
)
def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
try:
return VertexGemmaContainerError.model_validate(predictions)
@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
def _unwrap_predictions_response(
self,
response_json: dict[str, Any],
response_json: _PredictionsEnvelope,
) -> dict[str, object]:
"""
Unwrap the Vertex Gemma predictions format to OpenAI format.
@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
message=f"Request failed: {response.text}",
)
response_json: Final = response.json()
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
# Unwrap predictions to get OpenAI-compatible response
openai_response: Final = self._unwrap_predictions_response(response_json)
@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
message=f"Request failed: {response.text}",
)
response_json: Final = response.json()
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
# Unwrap predictions to get OpenAI-compatible response
openai_response: Final = self._unwrap_predictions_response(response_json)

View file

@ -3,6 +3,7 @@ from types import UnionType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
import httpx
from pydantic import ConfigDict, TypeAdapter
from pydantic import fields as pyd_fields
import litellm
@ -37,6 +38,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class _EventModelClass(Protocol):
@property
@ -420,7 +423,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
@staticmethod
def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]:
return raw_response.json()
return _RESPONSE_BODY.validate_python(raw_response.json())
@staticmethod
def _annotation_origin(annotation: object) -> object:

View file

@ -28853,6 +28853,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 1.25e-06,
@ -28910,6 +28911,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_image_token_batches": 6e-05,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
@ -28963,12 +28965,13 @@
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_image_token_batches": 6e-05,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_priority": 2.16e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"output_cost_per_token_batches": 6e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/images/generations",
@ -29014,6 +29017,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_image_token_batches": 3e-05,
"output_cost_per_token": 3e-06,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
@ -29059,9 +29063,10 @@
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_image_token_batches": 3e-05,
"output_cost_per_token": 3e-06,
"output_cost_per_token_batches": 1.5e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/images/generations",
@ -29106,6 +29111,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"output_cost_per_token_flex": 7.5e-07,
@ -29672,10 +29678,12 @@
},
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"input_cost_per_token_batches": 6.25e-07,
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06,
"output_cost_per_token_above_200k_tokens_priority": 2.7e-05,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_flex": 5e-06,
@ -29690,6 +29698,7 @@
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_above_200k_tokens_priority": 7.2e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_flex": 1e-06,
@ -29701,6 +29710,7 @@
"output_cost_per_image": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"output_cost_per_token_above_200k_tokens_batches": 9e-06,
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"output_cost_per_token_batches": 6e-06,
"output_cost_per_token_flex": 6e-06,
@ -30229,6 +30239,7 @@
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"input_cost_per_token_above_200k_tokens_priority": 7.2e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_flex": 1e-06,
@ -30240,6 +30251,7 @@
"output_cost_per_image": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"output_cost_per_token_above_200k_tokens_batches": 9e-06,
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"output_cost_per_token_batches": 6e-06,
"output_cost_per_token_flex": 6e-06,
@ -51910,6 +51922,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"output_cost_per_reasoning_token": 2.5e-06,
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_batches": 1.25e-06,
@ -51968,6 +51981,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_image_token_batches": 6e-05,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
@ -52002,13 +52016,14 @@
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_image_token_batches": 6e-05,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_priority": 2.16e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"output_cost_per_token_batches": 6e-06,
"supports_reasoning": false,
"source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image",
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/images/generations",
@ -52034,6 +52049,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_image_token_batches": 3e-05,
"output_cost_per_token": 3e-06,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
@ -52061,10 +52077,11 @@
"mode": "image_generation",
"output_cost_per_image": 0.0672,
"output_cost_per_image_token": 6e-05,
"output_cost_per_image_token_batches": 3e-05,
"output_cost_per_token": 3e-06,
"output_cost_per_token_batches": 1.5e-06,
"supports_reasoning": false,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/images/generations",
@ -52089,6 +52106,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"output_cost_per_token_flex": 7.5e-07,
@ -61986,6 +62004,39 @@
"supports_reasoning": true,
"supports_vision": false
},
"tencent/seedream-image-v5.0-flash": {
"litellm_provider": "tencent",
"mode": "image_generation",
"output_cost_per_image": 0.018,
"output_cost_per_image_token": 1.6e-06,
"source": "https://cloud.tencent.com/document/product/1823/130055",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"tencent/seedream-image-v5.0-lite": {
"litellm_provider": "tencent",
"mode": "image_generation",
"output_cost_per_image": 0.035,
"output_cost_per_image_token": 1.6e-06,
"source": "https://www.tencentcloud.com/document/product/1300/78937",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"tencent/seedream-image-v5.0-pro": {
"litellm_provider": "tencent",
"mode": "image_generation",
"output_cost_per_image": 0.09,
"output_cost_per_image_token": 1.6e-06,
"source": "https://www.tencentcloud.com/document/product/1300/78937",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"cognition/swe-1.6": {
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.5e-06,
@ -74367,6 +74418,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"output_cost_per_token": 7.5e-06,
"output_cost_per_token_batches": 3.75e-06,
"output_cost_per_token_priority": 1.35e-05,
@ -74412,6 +74464,7 @@
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"output_cost_per_token": 7.5e-06,
"output_cost_per_token_batches": 3.75e-06,
"output_cost_per_token_priority": 1.35e-05,

View file

@ -3793,8 +3793,8 @@
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"image_generations": true,
"image_edits": true,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,

View file

@ -26,6 +26,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.a2a.version_convert import (
A2AVersion,
@ -320,7 +321,7 @@ async def _a2a_sse_event_source(
"error": {"code": -32603, "message": resp.reason_phrase},
}
return
async for line in resp.aiter_lines():
async for line in aiter_sse_lines(resp.aiter_text()):
stripped = line.strip()
if not stripped.startswith("data:"):
continue

View file

@ -14,6 +14,8 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends
_MESSAGE_START_EVENT: Final = "message_start"
_MESSAGE_START_MARKER: Final = b"message_start"
_SSE_DATA_FIELD: Final = "data:"
@ -52,7 +54,7 @@ def _restamped_data_line(line: str, requested_model: str) -> str | None:
def _restamped_frame(frame: str, requested_model: str) -> str | None:
lines: Final = frame.splitlines(keepends=True)
lines: Final = split_sse_lines_keepends(frame)
restamped: Final = tuple(_restamped_data_line(line, requested_model) for line in lines)
if all(line is None for line in restamped):
return None

View file

@ -3,9 +3,12 @@ from collections.abc import Mapping
from typing import Any, Final
import requests
from pydantic import ConfigDict, TypeAdapter
from .exceptions import NotFoundError, UnauthorizedError
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class ModelsManagementClient:
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
@ -109,7 +112,7 @@ class ModelsManagementClient:
try:
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
return _JSON_OBJECT.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)
@ -145,7 +148,7 @@ class ModelsManagementClient:
try:
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
return _JSON_OBJECT.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)
@ -287,7 +290,7 @@ class ModelsManagementClient:
try:
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
return _JSON_OBJECT.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)

View file

@ -139,8 +139,8 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
)
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
from litellm.router import Router
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.types.extraction import ExtractionResponse
from litellm.types.guardrails import GuardrailEventHooks
@ -266,6 +266,8 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
AnthropicStreamModelRestamper,
)
from litellm.proxy.litellm_pre_call_utils import (
ROUTER_SCHEDULED_ROUTE_TYPES,
SCHEDULER_PRIORITY_HEADER,
LiteLLMProxyRequestSetup,
add_litellm_data_to_request,
get_metadata_variable_name,
@ -2478,14 +2480,20 @@ class ProxyBaseLLMRequestProcessing:
self.data["model"] = alias_target
aliased_model: Final = self.data.get("model")
if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
self.data.update(
llm_router.priority_header_update(
model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
)
)
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
self.data["litellm_call_id"] = litellm_call_id
bind_telemetry_call_id(litellm_call_id)
DDSpanTagger.tag_call_id(self.data.get("litellm_call_id"))
DDSpanTagger.tag_call_id(litellm_call_id)
DDSpanTagger.tag_request(
user_api_key_dict=user_api_key_dict,
requested_model=self.data.get("model"),
requested_model=aliased_model,
)
### AUTO STREAM USAGE TRACKING ###

View file

@ -0,0 +1,269 @@
from __future__ import annotations
import hashlib
import json
from collections.abc import AsyncGenerator, Callable
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter
from typing_extensions import assert_never
from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent
from litellm.proxy.db.db_span import db_span
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
if TYPE_CHECKING:
from prisma import Prisma
from litellm.proxy.utils import PrismaClient
AccountingEvent: TypeAlias = InboundUsageEvent | OutboundUsageEvent | CostEvent
_JSON: Final = TypeAdapter(dict[str, JsonValue])
_INSTANT: Final[TypeAdapter[datetime]] = TypeAdapter(AwareDatetime)
_STRINGS: Final = TypeAdapter(tuple[dict[str, str], ...])
_TIMEOUT: Final = timedelta(seconds=10)
class AccountingConflictError(ValueError):
pass
_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
@dataclass(frozen=True, slots=True)
class _AcceptedEvent:
event_id: str
kind: EventKind
source_event_id: str | None
occurred_at: datetime
payload: str
payload_hash: str
targets: tuple[str, ...]
schema_version: int
version: int
def _json(value: object) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
def _prepare(
event: AccountingEvent,
targets: tuple[DeliveryTarget, ...],
envelope: str,
schema_version: int,
version: int,
) -> _AcceptedEvent:
if type(schema_version) is not int or type(version) is not int or min(schema_version, version) < 1:
raise ValueError("Accounting versions must be positive integers")
validated: Final = type(event).model_validate_json(event.model_dump_json())
payload: Final = _JSON.validate_json(envelope)
if payload.get("event") != validated.model_dump(mode="json"):
raise ValueError("Accounting envelope must contain the matching public event")
destinations: Final = tuple(sorted(frozenset(DeliveryTarget(target).value for target in targets)))
serialized: Final = _json(payload)
identity: Final = _json((schema_version, version, destinations, payload))
kind: Final = (
EventKind.COST
if isinstance(validated, CostEvent)
else EventKind.INBOUND_USAGE
if isinstance(validated, InboundUsageEvent)
else EventKind.OUTBOUND_USAGE
)
return _AcceptedEvent(
event_id=validated.event_id,
kind=kind,
source_event_id=validated.usage_event_id if isinstance(validated, CostEvent) else None,
occurred_at=validated.occurred_at if isinstance(validated, CostEvent) else validated.ended_at,
payload=serialized,
payload_hash=hashlib.sha256(identity.encode()).hexdigest(),
targets=destinations,
schema_version=schema_version,
version=version,
)
_INSERT_EVENT: Final = """
INSERT INTO "LiteLLM_AccountingEvent"
(event_id, kind, source_event_id, schema_version, payload_hash, occurred_at, payload)
VALUES ($1, $2::"AccountingEventKind", $3, $4::int, $5, $6::timestamp, $7::jsonb)
ON CONFLICT (event_id) DO NOTHING RETURNING event_id
"""
_INSERT_DELIVERIES: Final = """
INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
SELECT $1, value::"AccountingDeliveryTarget", $3::int FROM jsonb_array_elements_text($2::jsonb)
"""
_LOCK_SOURCE: Final = """
SELECT event_id FROM "LiteLLM_AccountingEvent"
WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE
"""
_EXISTING_COST: Final = """
SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_event_id = $1
"""
_LOCK_EVENT: Final = """
SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
"""
_COMPLETED_FAMILY: Final = """
root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
AND root.recorded_at < $1::timestamp
AND (root.kind = 'inbound_usage' OR EXISTS (
SELECT 1 FROM "LiteLLM_AccountingEvent" AS cost WHERE cost.source_event_id = root.event_id AND cost.kind = 'cost'
))
AND NOT EXISTS (
SELECT 1 FROM "LiteLLM_AccountingEvent" AS member
WHERE (member.event_id = root.event_id OR (member.kind = 'cost' AND member.source_event_id = root.event_id))
AND (member.recorded_at >= $1::timestamp
OR EXISTS (SELECT 1 FROM "LiteLLM_AccountingDelivery" AS delivery
WHERE delivery.event_id = member.event_id AND delivery.state != 'applied'))
)
"""
_RETENTION_CANDIDATES: Final = (
'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE '
+ _COMPLETED_FAMILY
+ " ORDER BY root.recorded_at, root.event_id LIMIT $2::int FOR UPDATE OF root SKIP LOCKED"
)
_ELIGIBLE_GROUP: Final = (
'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE root.event_id = $2 AND ' + _COMPLETED_FAMILY
)
_DELETE_DELIVERIES: Final = """
DELETE FROM "LiteLLM_AccountingDelivery" WHERE event_id IN (
SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
)
"""
_DELETE_EVENTS: Final = """
DELETE FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
"""
_Conflict: TypeAlias = Literal["missing_source", "competing_cost", "identity_conflict"]
async def _accept(db: Prisma, record: _AcceptedEvent) -> Literal["inserted", "duplicate"] | _Conflict:
if record.source_event_id is not None:
source: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_SOURCE, record.source_event_id))
if not source:
return "missing_source"
costs: Final = _STRINGS.validate_python(tuple(await db.query_raw(_EXISTING_COST, record.source_event_id)))
if any(cost["event_id"] != record.event_id for cost in costs):
return "competing_cost"
inserted: Final = _STRINGS.validate_python(
await db.query_raw(
_INSERT_EVENT,
record.event_id,
record.kind.value,
record.source_event_id,
record.schema_version,
record.payload_hash,
record.occurred_at.astimezone(timezone.utc).isoformat(),
record.payload,
)
)
if not inserted:
existing: Final = _STRINGS.validate_python(tuple(await db.query_raw(_LOCK_EVENT, record.event_id)))
if not existing or existing[0]["payload_hash"] != record.payload_hash:
return "identity_conflict"
return "duplicate"
await db.execute_raw(_INSERT_DELIVERIES, record.event_id, _json(record.targets), record.version)
return "inserted"
async def _delete_completed(db: Prisma, event_id: str, cutoff: str) -> int:
if not _STRINGS.validate_python(await db.query_raw(_ELIGIBLE_GROUP, cutoff, event_id)):
return 0
await db.execute_raw(_DELETE_DELIVERIES, event_id)
return await db.execute_raw(_DELETE_EVENTS, event_id)
class AccountingLedger:
def __init__(self, transaction: Callable[[], _Transaction]) -> None:
self._transaction: Final = transaction
@classmethod
def for_client(cls, client: PrismaClient) -> AccountingLedger:
from prisma.client import TransactionManager
managers: Final = TypeAdapter(TransactionManager, config=ConfigDict(arbitrary_types_allowed=True))
@asynccontextmanager
async def transaction() -> AsyncGenerator[Prisma]:
primary: Final = writer_wrapper(client.db)
manager: Final = managers.validate_python(primary.tx(timeout=_TIMEOUT))
async with db_span("accounting_ledger", "LiteLLM_AccountingEvent"), manager as db:
await db.execute_raw("SET LOCAL statement_timeout = 5000")
await db.execute_raw("SET LOCAL lock_timeout = 1000")
yield db
return cls(transaction)
async def insert_event(
self,
event: AccountingEvent,
targets: tuple[DeliveryTarget, ...],
*,
envelope: str,
schema_version: int = 1,
version: int = 1,
) -> Literal["inserted", "duplicate"]:
"""Accept T5/T6's serialized envelope, containing its public event under `event`.
The immutable acceptance hash includes the target set and delivery version.
Changing either for an accepted event conflicts. T7 must provide a separate
add-target or new-version workflow that preserves receipts and prevents
duplicate contributions; redelivery cannot silently create or reset work.
The Prisma adapter opens one interactive transaction per event, including
two SET LOCAL statements and a source-row lock for costs. I/O is async,
but awaiting this method inline adds commit latency per event, potentially
several times per request. T5 owns bounded asynchronous capture and an
explicit batching strategy; looping over this method does not batch commits.
Success acknowledges commit, independently of HTTP response acknowledgement.
"""
record: Final = _prepare(event, targets, envelope, schema_version, version)
async with self._transaction() as db:
result: Final = await _accept(db, record)
match result:
case "inserted" | "duplicate":
return result
case "missing_source":
raise AccountingConflictError("Cost source must be a durably accepted outbound usage event")
case "competing_cost":
raise AccountingConflictError("Usage event already has an authoritative cost")
case "identity_conflict":
raise AccountingConflictError("Event identity conflicts with its accepted content or delivery set")
case _:
assert_never(result)
async def purge_completed(
self,
*,
redelivery_before: datetime | None = None,
recovery_before: datetime | None = None,
limit: int = 100,
) -> int:
"""Delete at most `limit` completed families once both retention owners permit it.
T8 must supply a coherent recovery watermark including outstanding holds.
Omitting either watermark disables deletion. Each usage row and its cost
remain together so a retained charge never loses its source or vice versa.
Applied receipts are deleted with the family, ending its retained dedup
protection. The caller must guarantee events recorded before
redelivery_before cannot be redelivered through supported paths.
T8's recovery_before must also protect checkpoints,
outstanding holds and unfinished settlements; age alone is insufficient.
"""
if type(limit) is not int or not 1 <= limit <= 1000:
raise ValueError("Accounting retention batch size must be between 1 and 1000")
if redelivery_before is None or recovery_before is None:
return 0
cutoff: Final = min(
_INSTANT.validate_python(redelivery_before).astimezone(timezone.utc),
_INSTANT.validate_python(recovery_before).astimezone(timezone.utc),
).isoformat()
async with self._transaction() as db:
roots: Final = _STRINGS.validate_python(tuple(await db.query_raw(_RETENTION_CANDIDATES, cutoff, limit)))
deleted: Final = tuple([await _delete_completed(db, root["event_id"], cutoff) for root in roots])
return sum(deleted)

View file

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:

View file

@ -169,7 +169,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
from litellm.types.llms.openai import OpenAIModerationResponse
return OpenAIModerationResponse(**response.json())
return OpenAIModerationResponse.model_validate(response.json())
def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
"""

View file

@ -27,7 +27,7 @@ from litellm.constants import (
HEALTH_CHECK_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
StrategyRouterDependency,
classify_strategy_router_model,
strategy_router_dependencies,

View file

@ -37,8 +37,8 @@ from litellm.proxy.spend_tracking.savings import (
_effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner
_proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner
)
from litellm.router_strategy.auto_router.baseline_request import baseline_request
from litellm.router_strategy.complexity_router.context_compaction import compaction_applied
from litellm.router_utils.baseline_request import baseline_request
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import BaselineRouteStamp
from litellm.types.utils import CallTypes, ModelInfo, Usage

View file

@ -19,6 +19,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import Span
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.bedrock.common_utils import get_bedrock_base_model, get_bedrock_cross_region_inference_regions
@ -30,7 +31,6 @@ from litellm.proxy.hooks.model_budget_shared_windows import (
SharedWindows,
)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload

View file

@ -118,6 +118,9 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
_SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority"
ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"})
# W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
# e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
_TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
@ -688,7 +691,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t
def _strip_untrusted_request_header_controls(
headers: Any,
headers: object,
*,
allow_client_message_redaction_opt_out: bool = False,
) -> None:
@ -1692,7 +1695,7 @@ class LiteLLMProxyRequestSetup:
@staticmethod
def add_internal_user_from_user_mapping(
general_settings: dict | None,
general_settings: Mapping[str, object] | None,
user_api_key_dict: UserAPIKeyAuth,
headers: dict,
) -> UserAPIKeyAuth:
@ -1711,7 +1714,7 @@ class LiteLLMProxyRequestSetup:
return user_api_key_dict
@staticmethod
def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None:
"""
Get the user from the specified header if `general_settings.user_header_name` is set.
"""
@ -1732,7 +1735,9 @@ class LiteLLMProxyRequestSetup:
return user
@staticmethod
def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
def get_openai_org_id_from_headers(
headers: dict, general_settings: Mapping[str, object] | None = None
) -> str | None:
"""
Get the OpenAI Org ID from the headers.
"""
@ -1823,7 +1828,7 @@ class LiteLLMProxyRequestSetup:
headers: dict,
request_data: Mapping[str, object],
user_api_key_dict: UserAPIKeyAuth,
general_settings: dict[str, object] | None = None,
general_settings: Mapping[str, object] | None = None,
) -> LitellmDataForBackendLLMCall:
"""
- Adds user from headers
@ -3285,7 +3290,7 @@ def _per_user_credential_names_for_groups(
GITHUB_COPILOT_AUTH_TYPE_KEY,
GITHUB_COPILOT_PER_USER_AUTH_TYPE,
)
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model
names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment
for group in model_groups:

View file

@ -60,16 +60,16 @@ from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.router_strategy.auto_router.auto_router_model_naming import (
StrategyRouterDependencyRole,
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_strategy.complexity_router import ComplexityRouter
from litellm.router_strategy.complexity_router.request_models import (
AutoRouterRoutingTestRequest,
RequestComplexityRouterConfig,
)
from litellm.router_utils.auto_router_model_naming import (
StrategyRouterDependencyRole,
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_TURN_VALVE,
@ -231,9 +231,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa
return _ShadowEvalAttemptRepository(prisma_client).table
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]:
async with db_span("auto_router_report_query", sql_relation(query)):
return await prisma_client.db.query_raw(query, *args)
return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args))
async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int:
@ -408,7 +411,7 @@ async def validate_complexity_router_config(
auto_router_environment_reference_error(data.complexity_router_config, user_api_key_dict)
)
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
validate_complexity_router_config_write,
)

View file

@ -131,6 +131,23 @@ from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import ModelTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.router import Router
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
carries_complexity_router_settings,
count_capability_routers,
gated_capabilities_of,
is_complexity_router_model,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
validate_strategy_router_model_write,
)
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
is_mutable_tuned_candidate,
tuning_quota_violation,
)
from litellm.router_strategy.complexity_router import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
ClassificationRubric,
@ -144,20 +161,6 @@ from litellm.router_strategy.complexity_router import (
normalize_classification_prompt,
)
from litellm.router_strategy.complexity_router.config import resolve_complexity_router_config_write
from litellm.router_utils.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
carries_complexity_router_settings,
count_capability_routers,
gated_capabilities_of,
is_complexity_router_model,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
validate_strategy_router_model_write,
)
from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.bedrock import AwsSessionTag
from litellm.types.proxy.management_endpoints.model_management_endpoints import (

View file

@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U
import inspect
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, get_args
from typing import Final, get_args
from fastapi import APIRouter, Depends
from pydantic import Field
@ -32,7 +32,7 @@ router: Final = APIRouter()
class RouterSettingsResponse(LiteLLMBaseModel):
fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
current_values: dict[str, Any] = Field(description="Current values of router settings")
current_values: dict[str, object] = Field(description="Current values of router settings")
routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
routing_group_strategies: tuple[str, ...] = Field(
description="Strategies supported when constructing a routing group"

View file

@ -7,14 +7,14 @@ from typing import Final
from pydantic import Json, TypeAdapter, ValidationError
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
capability_limit_violation,
classify_strategy_router_model,
count_capability_routers,
gated_capabilities_of,
)
from litellm.router_utils.auto_router_tuning_baseline import (
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
is_mutable_tuned_candidate,
mutable_tuned_identities,
tuning_quota_violation,

View file

@ -34,13 +34,16 @@ from litellm.repositories.prisma_protocols import DatabaseClient
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import TeamMembershipRepository
from litellm.router import Router
from litellm.router_strategy.auto_router.auto_router_model_naming import (
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_strategy.complexity_router.config import (
ComplexityRouterConfigWrite,
normalize_classifier_config_aliases,
resolve_complexity_router_config_write,
)
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import Deployment, updateDeployment

View file

@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
)
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
from litellm.repositories.team_repository import TeamRepository
if TYPE_CHECKING:
from prisma import Prisma
@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool:
async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone:
team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id})
team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id})
if team_row is None:
return TeamGone(team_id=team_id)
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())

View file

@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict:
if decrypted_json is None:
raise ValueError("Failed to decrypt state parameter")
state_data: Final = json.loads(decrypted_json)
state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json))
return state_data
_MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
_DECODED_JSON: Final = TypeAdapter(object)
_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
class _ManagedAuthorizationCode(LiteLLMBaseModel):

View file

@ -29,6 +29,7 @@ from litellm.llms.anthropic.messages.stream_assembly import (
split_sse_chunk_into_events,
)
from litellm.llms.bedrock.common_utils import BEDROCK_INVOKE_TRACE_FIELD
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
@ -61,7 +62,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
def _event_payload(event: str) -> Mapping[str, object] | None:
data_lines: Final = tuple(
line.strip()[len("data:") :].strip() for line in event.splitlines() if line.strip().startswith("data:")
line.strip()[len("data:") :].strip() for line in split_sse_lines(event) if line.strip().startswith("data:")
)
if not data_lines or data_lines[0] in ("", "[DONE]"):
return None
@ -362,7 +363,7 @@ class AnthropicPassthroughLoggingHandler:
) -> str | None:
for raw in all_chunks:
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
for line in text.splitlines():
for line in split_sse_lines(text):
if not line.startswith("data:"):
continue
try:
@ -402,7 +403,7 @@ class AnthropicPassthroughLoggingHandler:
"""
for raw in reversed(all_chunks):
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
for line in reversed(text.splitlines()):
for line in reversed(split_sse_lines(text)):
if not line.startswith("data:"):
continue
try:

View file

@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
)
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.passthrough_endpoints.tinyfish import (
@ -81,7 +82,7 @@ def sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> bool:
def run_id_from_sse_frames(frames: bytes) -> str | None:
return _run_id_from_sse_chunks(frames.decode("utf-8", errors="replace").splitlines())
return _run_id_from_sse_chunks(split_sse_lines(frames.decode("utf-8", errors="replace")))
def resolve_tinyfish_agent_api_base() -> str:

View file

@ -51,6 +51,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
can_access_resource,
resolve_resource_owner_id,
)
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
@ -830,7 +831,7 @@ _SSE_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue])
def _first_streamed_response(frames: bytes) -> tuple[str, Mapping[str, JsonValue]] | None:
for line in frames.decode("utf-8", errors="replace").splitlines():
for line in split_sse_lines(frames.decode("utf-8", errors="replace")):
if not line.startswith(_SSE_DATA_PREFIX):
continue
try:

View file

@ -30,6 +30,7 @@ from pydantic import TypeAdapter
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.routing import BaseRoute, Route
from starlette.websockets import WebSocketState
from typing_extensions import ReadOnly
from websockets.asyncio.client import connect
from websockets.exceptions import (
ConnectionClosedError,
@ -175,8 +176,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool:
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
class _RegisteredPassThroughRoute(TypedDict):
endpoint_id: ReadOnly[str]
path: ReadOnly[str | None]
type: ReadOnly[str]
methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]]
auth: ReadOnly[bool]
passthrough_params: ReadOnly[Mapping[str, object]]
# Global registry to track registered pass-through routes and prevent memory leaks
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {}
def get_response_body(response: httpx.Response) -> dict | None:
@ -2267,7 +2277,7 @@ def create_pass_through_route(
status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.",
)
target_params: Final = {
target_params: Final[dict[str, object]] = {
"target": target,
"custom_headers": custom_headers,
"forward_headers": _forward_headers,
@ -2299,7 +2309,9 @@ def create_pass_through_route(
# Ensure custom_headers is a dict. Botocore returns a HeadersDict
# for SigV4-prepared requests, which is a Mapping but not a dict.
headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {}
headers_dict: Final[dict[str, object]] = (
{**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {}
)
# Ensure query_params and custom_body are dicts or None
final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {}
@ -3132,9 +3144,7 @@ class SafeRouteAdder:
return f"{path}/{{subpath:path}}"
return None
def registered_route_methods(
route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]],
) -> tuple[tuple[str, str], ...]:
def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]:
starlette_path: Final = registered_route_path(route_info)
methods: Final = route_info.get("methods")
if starlette_path is None or not isinstance(methods, (list, tuple)):
@ -3490,7 +3500,7 @@ class InitPassThroughEndpointHelpers:
return (comparison_route, *forwarded)
@staticmethod
def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None:
def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None:
"""Get passthrough params for a given route and optionally filter by HTTP method"""
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
for key in _registered_pass_through_routes:

View file

@ -168,12 +168,7 @@ from litellm.proxy.common_utils.token_count_settings import (
)
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
from litellm.router_utils.access_windows import access_windows_config_error
from litellm.router_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
get_hidden_params_dict,
)
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
capability_limit_violation,
@ -181,12 +176,17 @@ from litellm.router_utils.auto_router_model_naming import (
count_capability_routers,
validate_complexity_router_config_placement,
)
from litellm.router_utils.auto_router_tuning_baseline import (
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
TUNING_BASELINE_PARAM_NAME,
mutable_tuned_identities,
snapshot_tuning_baselines,
tuning_limit_violation,
)
from litellm.router_utils.access_windows import access_windows_config_error
from litellm.router_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
get_hidden_params_dict,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.routing_groups import parse_routing_groups
from litellm.types.caching import RedisPipelineIncrementOperation
@ -666,7 +666,11 @@ from litellm.proxy.list_api.common import (
problem_response,
request_validation_problem,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
from litellm.proxy.litellm_pre_call_utils import (
SCHEDULER_PRIORITY_HEADER,
LiteLLMProxyRequestSetup,
add_litellm_data_to_request,
)
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
rust_control_plane_router,
)
@ -18481,6 +18485,11 @@ async def async_queue_request(
raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value})
router: Final = llm_router
data.update(
router.priority_header_update(
model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
)
)
queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data)
if "stream" in data and data["stream"] is True: # use generate_responses to stream responses

View file

@ -18,6 +18,7 @@ from starlette.types import Message
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
@ -81,7 +82,7 @@ class _StreamsBody(Protocol):
def _sse_frame_data(frame: str) -> str | None:
return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None)
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
async def _never_receive() -> Message:

View file

@ -6,7 +6,7 @@ from datetime import datetime, timedelta
from types import MappingProxyType
from typing import Final, Literal, TypeVar
from pydantic import TypeAdapter
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel):
last_owner: str | None = None
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
_SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...])
_DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...])
@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout(
await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL)
for setting in planner_settings:
await transaction.execute_raw(setting)
return await transaction.query_raw(sql, *params)
return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params))
async def _reverse_hash_key_metadata(

View file

@ -165,6 +165,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._upstream_exhausted: bool = False
self._response_id_primed: bool = False
self._pending_tool_events: list[BaseLiteLLMOpenAIResponseObject] = []
self._tool_calls_held_behind_text: tuple[object, ...] = ()
self._tool_output_index_by_call_id: dict[str, int] = {}
self._tool_args_by_call_id: dict[str, str] = {}
self._tool_item_id_by_call_id: dict[str, str] = {} # mutable-ok: filled per call id as tool call events stream
@ -388,6 +389,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return ""
return self._tool_call_id_by_index.get(index, "")
def _hold_tool_calls_behind_text(self, tool_calls: object) -> None:
if isinstance(tool_calls, list):
self._tool_calls_held_behind_text = (*self._tool_calls_held_behind_text, *tool_calls)
def _release_held_tool_calls(self) -> tuple[object, ...]:
held: Final = self._tool_calls_held_behind_text
self._tool_calls_held_behind_text = ()
return held
def _queue_tool_call_delta_events(self, tool_calls: object) -> None:
"""
Convert chat-completions streaming `tool_calls` deltas into Responses API streaming events.
@ -1365,6 +1375,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if delta_content:
if not self.sent_message_item_added_event:
self._queue_message_item_added_events()
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
self._sequence_number += 1
text_delta_event: Final = OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
@ -1379,7 +1390,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# Priority 3: Handle tool call deltas (if any) -> queue events and emit them
# For each tool call delta, we emit events one at a time to match OpenAI's streaming behavior
if chunk.choices and hasattr(chunk.choices[0].delta, "tool_calls") and chunk.choices[0].delta.tool_calls:
self._queue_tool_call_delta_events(chunk.choices[0].delta.tool_calls)
self._queue_tool_call_delta_events([*self._release_held_tool_calls(), *chunk.choices[0].delta.tool_calls])
# Return one pending tool event at a time
if self._pending_tool_events:
return self._pending_tool_events.pop(0)

View file

@ -7,6 +7,29 @@
#
# Thank you ! We ❤️ you! - Krrish & Ishaan
"""
litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments.
Public surface. The router is moving behind a narrow interface; code outside
the router should end up using a Router through two kinds of members only:
- endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages,
the files, batches, assistants and vector store wrappers, ...), each a thin
wrapper around one call;
- entry points (being added) that replace direct access to internals: registry() for
deployment lookups, settings() for configuration reads, health() for
cooldowns, blocks and usage, select() to pick a deployment without calling
it, apply() for runtime changes, observe() for routing events,
register_plugin() for strategies and sub-routers, and retire() to release
the router.
Everything else on the class is internal. The members outside code uses today
are listed in tests/code_coverage_tests/router_surface_allowlist.json, and a CI
check (router_surface_check.py next to it) fails on any new outside use, so the
list only shrinks while callers move to the entry points. The same interface
is documented for the Rust core in litellm-rust/crates/router-core.
"""
import asyncio
import contextlib
import copy
@ -85,6 +108,15 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.batch_utils import (
batch_create_routing,
deployment_supported_endpoints,
get_router_metadata_variable_name,
is_batch_retrieve_call_type,
replace_model_in_jsonl,
should_replace_model_in_jsonl,
without_deployments_that_cannot_batch,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
@ -148,6 +180,15 @@ from litellm.llms.openai_like.model_info import (
get_openai_compatible_model_info,
)
from litellm.responses.bridge_check import keep_registered_mode
from litellm.router_strategy.auto_router.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
claimed_capabilities,
classify_strategy_router_model,
count_capability_routers,
)
from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.complexity_router.context_compaction import (
@ -187,24 +228,6 @@ from litellm.router_utils.add_retry_fallback_headers import (
replace_complexity_router_headers,
response_total_token_count,
)
from litellm.router_utils.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
claimed_capabilities,
classify_strategy_router_model,
count_capability_routers,
)
from litellm.router_utils.batch_utils import (
batch_create_routing,
deployment_supported_endpoints,
get_router_metadata_variable_name,
is_batch_retrieve_call_type,
replace_model_in_jsonl,
should_replace_model_in_jsonl,
without_deployments_that_cannot_batch,
)
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
from litellm.router_utils.clientside_credential_handler import (
get_dynamic_litellm_params,
@ -359,6 +382,7 @@ from litellm.router_utils.request_priority import (
InvalidPriority,
parse_default_priority,
priority_error,
priority_header_fields,
request_drops_params,
resolve_request_priority,
unqueueable_priority_message,
@ -1369,7 +1393,7 @@ class Router:
},
]
router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}])
```
"""
@ -1595,6 +1619,10 @@ class Router:
if default_fallbacks is not None or litellm.default_fallbacks is not None:
_fallbacks = default_fallbacks or litellm.default_fallbacks
if isinstance(_fallbacks, str):
raise TypeError(
f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']"
)
if self.fallbacks is not None:
self.fallbacks.append({"*": _fallbacks})
else:
@ -2838,7 +2866,7 @@ class Router:
self._initialize_core_endpoints()
self._initialize_specialized_endpoints()
def validate_fallbacks(self, fallback_param: list | None):
def validate_fallbacks(self, fallback_param: Sequence[Mapping[str, object]] | None):
"""
Validate the fallbacks parameter.
"""
@ -2851,6 +2879,13 @@ class Router:
raise ValueError(
f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."
)
((model_group, targets),) = fallback_dict.items()
if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=[fallback_dict]):
continue
raise TypeError(
f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. "
f"Use {{'{model_group}': ['{targets}']}}"
)
def _add_encrypted_content_affinity_check(self, enable_global_affinity: bool) -> None:
def _move_before_deployment_affinity(
@ -5411,14 +5446,22 @@ class Router:
request_priority: Final = resolve_request_priority(
requested=kwargs.get("priority"),
default_priority=self.default_priority,
drops_params=lambda: request_drops_params(
kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
),
drops_params=lambda: self._drops_params(model=model, kwargs=kwargs),
)
if isinstance(request_priority, InvalidPriority):
raise priority_error(model=model, message=request_priority.message)
return request_priority, {key: value for key, value in kwargs.items() if key != "priority"}
def priority_header_update(
self, model: str, kwargs: Mapping[str, object], header: str | None
) -> Mapping[str, int | str]:
return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs))
def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
return request_drops_params(
kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
)
async def _schedule_factory(
self,
model: str,
@ -16251,7 +16294,7 @@ class Router:
to the deployment that actually served the request. Every attempt therefore
writes or clears, never just writes.
"""
from litellm.router_utils.baseline_request import capture_baseline_parameters
from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters
from litellm.types.router import BaselineRouteStamp
phase_attributes(routing_decision_attributes(routing_decision))

View file

@ -36,12 +36,12 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import RedisCache, RedisPipelineIncrementOperation, log_redis_failure
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
from litellm.router_strategy.tag_based_routing import get_tags_from_request_kwargs
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.router_utils.cooldown_callbacks import (
get_prometheus_logger_from_callbacks,
)

View file

@ -10,7 +10,7 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import log_redis_failure
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60

View file

@ -10,7 +10,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed

View file

@ -13,9 +13,9 @@ from litellm import ModelResponse, token_counter, verbose_logger
from litellm._internal_context import with_service_target
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import LiteLLMPydanticObjectBase

View file

@ -9,7 +9,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.utils import LiteLLMPydanticObjectBase
from litellm.utils import print_verbose

Some files were not shown because too many files have changed in this diff Show more