mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge branch 'main' into litellm_fix_passthrough_failure_model_from_url
This commit is contained in:
commit
80dfcc1eee
203 changed files with 6614 additions and 922 deletions
3
.github/workflows/test-linting.yml
vendored
3
.github/workflows/test-linting.yml
vendored
|
|
@ -561,6 +561,9 @@ jobs:
|
|||
- name: router_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
|
||||
- name: router_surface_check
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
|
||||
|
||||
- name: test_chat_completion_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
|
||||
|
|
|
|||
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -245,6 +245,7 @@ jobs:
|
|||
tests/unit/llms/clarifai
|
||||
tests/unit/llms/claude_code
|
||||
tests/unit/llms/cloudflare
|
||||
tests/unit/llms/codestral
|
||||
tests/unit/llms/codex
|
||||
tests/unit/llms/cohere
|
||||
tests/unit/llms/cometapi
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
needs an approving review from the spend-log owner before it can merge.
|
||||
|
||||
CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
|
||||
so a path rule would put every migration on the owner. This gate reads the diff instead: when no
|
||||
added line in the watched files mentions the table it passes at once, otherwise it passes only once
|
||||
the owner's latest review on the pull request is an approval that still covers every spend-log line
|
||||
in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never
|
||||
saw). Reviews do not trigger the workflow, so the check is re-run by hand after approving.
|
||||
so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
|
||||
hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
|
||||
hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
|
||||
only once the owner's latest review on the pull request is an approval that still covers every spend-log line
|
||||
in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
|
||||
Reviews do not trigger the workflow, so the check is re-run by hand after approving.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -15,8 +16,10 @@ import argparse
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
|
@ -29,6 +32,8 @@ WATCHED: Final = (
|
|||
MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
|
||||
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
|
||||
DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
|
||||
MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
|
||||
HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
|
||||
|
||||
|
||||
class Reviewer(BaseModel):
|
||||
|
|
@ -46,26 +51,71 @@ class Review(BaseModel):
|
|||
_REVIEWS: Final = TypeAdapter(list[Review])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Block:
|
||||
"""Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
|
||||
|
||||
base: range
|
||||
head: range
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Hunk:
|
||||
removed: tuple[tuple[int, str], ...]
|
||||
added: tuple[tuple[int, str], ...]
|
||||
|
||||
|
||||
def watched(path: str) -> bool:
|
||||
return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
|
||||
|
||||
|
||||
def _added_lines(section: str) -> tuple[str, ...]:
|
||||
body: Final = section.split("\n@@", 1)
|
||||
return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
|
||||
def model_block(schema: str) -> range:
|
||||
"""1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
|
||||
lines: Final = schema.splitlines()
|
||||
starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
|
||||
if not starts:
|
||||
return range(0)
|
||||
ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
|
||||
return range(starts[0], (ends[0] if ends else len(lines)) + 1)
|
||||
|
||||
|
||||
def spend_log_hits(diff: str) -> tuple[str, ...]:
|
||||
"""`path: added line` for every added line in a watched file that names the table."""
|
||||
def _hunks(section: str) -> tuple[Hunk, ...]:
|
||||
headers: Final = tuple(HUNK.finditer(section))
|
||||
ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
|
||||
return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
|
||||
|
||||
|
||||
def _hunk(header: re.Match[str], body: str) -> Hunk:
|
||||
lines: Final = body.splitlines()
|
||||
return Hunk(
|
||||
removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
|
||||
added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
|
||||
)
|
||||
|
||||
|
||||
def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
|
||||
"""`path: +line` or `path: -line` for every changed line that touches the table.
|
||||
|
||||
A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
|
||||
merge base for removed ones), a migration line when it is added and names the table."""
|
||||
sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
|
||||
return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
|
||||
|
||||
|
||||
def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
|
||||
pairs: Final = tuple((_path_of(section), section) for section in sections)
|
||||
paths: Final = tuple((_path_of(section), section) for section in sections)
|
||||
return tuple(
|
||||
(path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
|
||||
) # comprehension-ok: path then its lines
|
||||
hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
|
||||
) # comprehension-ok: path then its hits
|
||||
|
||||
|
||||
def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
|
||||
if path not in blocks:
|
||||
return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
|
||||
block: Final = blocks[path]
|
||||
removed: Final = tuple(
|
||||
f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
|
||||
) # comprehension-ok: hunk then its lines
|
||||
added: Final = tuple(
|
||||
f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
|
||||
) # comprehension-ok: hunk then its lines
|
||||
return removed + added
|
||||
|
||||
|
||||
def _path_of(section: str) -> str:
|
||||
|
|
@ -111,13 +161,25 @@ def added_diff(base: str, head: str) -> str:
|
|||
return result.stdout
|
||||
|
||||
|
||||
def file_at(revision: str, path: str) -> str:
|
||||
"""Contents of `path` at `revision`, empty when the file does not exist there."""
|
||||
result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
|
||||
return result.stdout if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
|
||||
return MappingProxyType(
|
||||
{path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
|
||||
)
|
||||
|
||||
|
||||
def fetched(commit: str) -> bool:
|
||||
result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
|
||||
return spend_log_hits(added_diff(base, commit)) if fetched(commit) else None
|
||||
return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
|
||||
|
||||
|
||||
class Arguments(BaseModel):
|
||||
|
|
@ -135,9 +197,12 @@ def main(argv: Sequence[str]) -> int:
|
|||
parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
|
||||
parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
|
||||
args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
|
||||
hits: Final = spend_log_hits(added_diff(args.base, args.head))
|
||||
hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
|
||||
if not hits:
|
||||
sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
|
||||
sys.stdout.write(
|
||||
"No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
|
||||
"nothing to review.\n"
|
||||
)
|
||||
return 0
|
||||
reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
|
||||
approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
|
||||
|
|
|
|||
4
litellm-rust/Cargo.lock
generated
4
litellm-rust/Cargo.lock
generated
|
|
@ -4393,6 +4393,10 @@ dependencies = [
|
|||
"rstest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-types"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,36 @@
|
|||
//! The model catalog: per-model pricing, context limits and capabilities.
|
||||
//!
|
||||
//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
|
||||
//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
|
||||
//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
|
||||
//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
|
||||
//! in `litellm-router-core`, which reads this catalog through its dynamic config.
|
||||
//!
|
||||
//! # Structure
|
||||
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
|
||||
//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
|
||||
//! and keeps every entry in file order with its raw fields.
|
||||
//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
|
||||
//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
|
||||
//! modalities and modes in `capabilities`).
|
||||
//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
|
||||
//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
|
||||
//! failing the parse.
|
||||
//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
|
||||
//! families. They are data in the file, not entries.
|
||||
//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
|
||||
//! lost most of its models is refused), and `validate_registry` checks every entry against the
|
||||
//! checked-in contract.
|
||||
//!
|
||||
//! # Lookups
|
||||
//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
|
||||
//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
|
||||
//! prefixes are not stripped here; callers decide which keys to try.
|
||||
//!
|
||||
//! # Readers
|
||||
//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
|
||||
//! performs I/O; loading and refreshing the file is the caller's job.
|
||||
|
||||
mod capabilities;
|
||||
mod catalog;
|
||||
mod error;
|
||||
|
|
|
|||
9
litellm-rust/crates/router-core/Cargo.toml
Normal file
9
litellm-rust/crates/router-core/Cargo.toml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
|
||||
|
||||
[dependencies]
|
||||
63
litellm-rust/crates/router-core/src/lib.rs
Normal file
63
litellm-rust/crates/router-core/src/lib.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks
|
||||
//! and cooldowns around the attempts.
|
||||
//!
|
||||
//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in
|
||||
//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the
|
||||
//! same interface. Until then it holds this overview and nothing else, so the interface is written
|
||||
//! down in code before either side changes.
|
||||
//!
|
||||
//! # The public surface
|
||||
//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else:
|
||||
//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the
|
||||
//! files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over
|
||||
//! `call(op, request)`.
|
||||
//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`,
|
||||
//! `observe()`, `register_plugin()` and `retire()`.
|
||||
//!
|
||||
//! Every other member that outside code uses today has a named replacement among those. The old
|
||||
//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which
|
||||
//! implements them over the entry points and warns on use. A CI check
|
||||
//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside
|
||||
//! code touches; the list may only shrink.
|
||||
//!
|
||||
//! # Structure
|
||||
//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern
|
||||
//! and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`,
|
||||
//! `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and
|
||||
//! `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing
|
||||
//! and capabilities are the model catalog (`litellm-model-catalog`), which the router reads
|
||||
//! through its dynamic config.
|
||||
//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The
|
||||
//! router never reads process globals. Static config (model list, strategy, retry, timeout and
|
||||
//! cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config
|
||||
//! (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read
|
||||
//! once per call.
|
||||
//! - `Settings`: one typed, frozen view of the configuration a caller may read.
|
||||
//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and
|
||||
//! Rust workers see the same state. `report(id, outcome)` records outcomes the router did not
|
||||
//! see itself (a health check, an interceptor that marks a deployment failed).
|
||||
//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the
|
||||
//! configured strategy, without calling it.
|
||||
//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing
|
||||
//! a new registry version.
|
||||
//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
|
||||
//! policy is exhausted.
|
||||
//!
|
||||
//! # Execution
|
||||
//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
|
||||
//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
|
||||
//! started attempt is never replayed on a different executor.
|
||||
//!
|
||||
//! # Hosts
|
||||
//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
|
||||
//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
|
||||
//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
|
||||
//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
|
||||
//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
|
||||
//! serves them in the Rust server, where an operation without a Rust implementation makes the
|
||||
//! config fail at build time, never partway through a request.
|
||||
//!
|
||||
//! # Plugins
|
||||
//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
|
||||
//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
|
||||
//! the current request), never the router itself.
|
||||
|
|
@ -19,6 +19,7 @@ from litellm.a2a_protocol.utils import (
|
|||
)
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
|
||||
RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33
|
||||
RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256
|
||||
|
|
@ -50,7 +51,7 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = (
|
|||
class _SSELineSource(Protocol):
|
||||
"""Minimal streaming-response surface used to read SSE lines."""
|
||||
|
||||
def aiter_lines(self) -> AsyncIterator[str]: ...
|
||||
def aiter_text(self) -> AsyncIterator[str]: ...
|
||||
|
||||
|
||||
def _filter_reserved_headers(
|
||||
|
|
@ -244,7 +245,7 @@ class BedrockAgentCoreA2ATransformation:
|
|||
Yields:
|
||||
Parsed JSON dicts from SSE data lines
|
||||
"""
|
||||
async for line in response.aiter_lines():
|
||||
async for line in aiter_sse_lines(response.aiter_text()):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
_IBM_CLOUD_IAM_URL: Final = "https://iam.cloud.ibm.com/identity/token"
|
||||
|
|
@ -74,7 +75,7 @@ class _WXORun(TypedDict, total=False):
|
|||
|
||||
|
||||
class _SSELineSource(Protocol):
|
||||
def aiter_lines(self) -> AsyncIterator[str]: ...
|
||||
def aiter_text(self) -> AsyncIterator[str]: ...
|
||||
|
||||
|
||||
class _WXOView(TypedDict, total=False):
|
||||
|
|
@ -231,7 +232,7 @@ class WatsonxOrchestrateHandler:
|
|||
async def _accumulate_wxo_sse_text(response: _SSELineSource) -> str:
|
||||
source: Final[_WXOView] = {"sse_source": response}
|
||||
accumulated_text = ""
|
||||
async for line in source["sse_source"].aiter_lines():
|
||||
async for line in aiter_sse_lines(source["sse_source"].aiter_text()):
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[5:].strip()
|
||||
|
|
|
|||
|
|
@ -274,7 +274,9 @@ class InMemoryCache(BaseCache):
|
|||
) -> list[float] | None:
|
||||
results: Final = []
|
||||
for increment in increment_list:
|
||||
result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs)
|
||||
result = await self.async_increment(
|
||||
increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs
|
||||
)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
|||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
CompletionTransformationBridge,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool
|
||||
from litellm.responses.sse_output_recovery import (
|
||||
parse_sse_json_chunk,
|
||||
|
|
@ -1351,7 +1352,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
recovered_output_items: Final[dict[int, dict[str, object]]] = {}
|
||||
recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
|
||||
|
||||
for chunk in raw_sse.splitlines():
|
||||
for chunk in split_sse_lines(raw_sse):
|
||||
parsed_chunk = parse_sse_json_chunk(chunk)
|
||||
if parsed_chunk is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
|
|
@ -131,9 +132,9 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
|
|||
self.model = model
|
||||
self.generate_content_provider_config = generate_content_provider_config
|
||||
self.litellm_metadata = litellm_metadata
|
||||
# Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
|
||||
# Gemini streamGenerateContent uses SSE line framing; iter_sse_lines keeps
|
||||
# large inlineData payloads (e.g. image/jpeg) intact within one event.
|
||||
self.stream_iterator = response.iter_lines()
|
||||
self.stream_iterator = iter_sse_lines(response.iter_text())
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
|
@ -182,9 +183,9 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
|
|||
self.model = model
|
||||
self.generate_content_provider_config = generate_content_provider_config
|
||||
self.litellm_metadata = litellm_metadata
|
||||
# Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
|
||||
# Gemini streamGenerateContent uses SSE line framing; aiter_sse_lines keeps
|
||||
# large inlineData payloads (e.g. image/jpeg) intact within one event.
|
||||
self.stream_iterator = response.aiter_lines()
|
||||
self.stream_iterator = aiter_sse_lines(response.aiter_text())
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
import importlib
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from collections.abc import Collection, Coroutine, Mapping
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -61,6 +61,7 @@ from litellm.types.utils import (
|
|||
ProviderRoutingHints,
|
||||
)
|
||||
from litellm.utils import (
|
||||
EXTRA_BODY_ROUTING_KEYS,
|
||||
ImageResponse,
|
||||
ProviderConfigManager,
|
||||
get_llm_provider,
|
||||
|
|
@ -466,7 +467,7 @@ def image_generation(
|
|||
litellm.LlmProviders.QWENCLOUD,
|
||||
litellm.LlmProviders.QWEN_AI_PLATFORM,
|
||||
litellm.LlmProviders.EDENAI,
|
||||
):
|
||||
) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None):
|
||||
if image_generation_config is None:
|
||||
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
|
||||
|
||||
|
|
@ -807,6 +808,32 @@ def image_variation(
|
|||
return response
|
||||
|
||||
|
||||
_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams))
|
||||
_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"})
|
||||
_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...])
|
||||
|
||||
|
||||
def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
|
||||
try:
|
||||
return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def _tokenhub_passthrough_edit_params(
|
||||
non_default_params: Mapping[str, object],
|
||||
extra_body: Mapping[str, object] | None,
|
||||
mapped_params: Collection[str],
|
||||
additional_drop_params: Collection[str] | None,
|
||||
) -> Mapping[str, object]:
|
||||
dropped: Final = frozenset(additional_drop_params or ())
|
||||
unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped
|
||||
return {
|
||||
**{key: value for key, value in non_default_params.items() if key not in unmapped_excluded},
|
||||
**{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS},
|
||||
}
|
||||
|
||||
|
||||
@client
|
||||
def image_edit(
|
||||
image: FileTypes | list[FileTypes] | None = None,
|
||||
|
|
@ -935,22 +962,24 @@ def image_edit(
|
|||
raise ValueError(f"image edit is not supported for {custom_llm_provider}")
|
||||
|
||||
local_vars.update(kwargs)
|
||||
provider_supported_params: Final[frozenset[str]] = frozenset(
|
||||
image_edit_provider_config.get_supported_openai_params(model)
|
||||
)
|
||||
# Get ImageEditOptionalRequestParams with only valid parameters
|
||||
image_edit_optional_params: Final[ImageEditOptionalRequestParams] = (
|
||||
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
|
||||
local_vars,
|
||||
provider_supported_params=frozenset(
|
||||
image_edit_provider_config.get_supported_openai_params(model)
|
||||
).intersection(non_default_params),
|
||||
provider_supported_params=provider_supported_params.intersection(non_default_params),
|
||||
)
|
||||
)
|
||||
# Get optional parameters for the responses API
|
||||
additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params"))
|
||||
image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit(
|
||||
model=model,
|
||||
image_edit_provider_config=image_edit_provider_config,
|
||||
image_edit_optional_params=image_edit_optional_params,
|
||||
drop_params=kwargs.get("drop_params"),
|
||||
additional_drop_params=kwargs.get("additional_drop_params"),
|
||||
additional_drop_params=additional_drop_params,
|
||||
)
|
||||
|
||||
if image_edit_provider_config.use_multipart_form_data() and (
|
||||
|
|
@ -1039,7 +1068,19 @@ def image_edit(
|
|||
image=images,
|
||||
prompt=prompt,
|
||||
image_edit_provider_config=image_edit_provider_config,
|
||||
image_edit_optional_request_params=image_edit_request_params,
|
||||
image_edit_optional_request_params={
|
||||
**image_edit_request_params,
|
||||
**(
|
||||
_tokenhub_passthrough_edit_params(
|
||||
non_default_params,
|
||||
extra_body if isinstance(extra_body, dict) else None,
|
||||
provider_supported_params,
|
||||
additional_drop_params,
|
||||
)
|
||||
if custom_llm_provider == litellm.LlmProviders.TENCENT.value
|
||||
else {}
|
||||
),
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import base64
|
||||
import os
|
||||
from collections.abc import Collection, Mapping
|
||||
from io import BufferedReader, BytesIO
|
||||
from typing import Any, Final, cast, get_type_hints
|
||||
from pathlib import Path
|
||||
from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
|
@ -16,7 +19,7 @@ class ImageEditRequestUtils:
|
|||
image_edit_provider_config: BaseImageEditConfig,
|
||||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
drop_params: bool | None = None,
|
||||
additional_drop_params: list[str] | None = None,
|
||||
additional_drop_params: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Get optional parameters for the image edit API.
|
||||
|
|
@ -145,3 +148,36 @@ class ImageEditRequestUtils:
|
|||
except Exception:
|
||||
# If anything goes wrong, default to PNG
|
||||
return FILE_MIME_TYPES[FileType.PNG]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SeekableBinaryReader(Protocol):
|
||||
def tell(self) -> int: ...
|
||||
|
||||
def seek(self, offset: int) -> int: ...
|
||||
|
||||
def read(self) -> bytes: ...
|
||||
|
||||
|
||||
def _read_image_bytes(image: object) -> bytes:
|
||||
if isinstance(image, bytes):
|
||||
return image
|
||||
if isinstance(image, tuple):
|
||||
return _read_image_bytes(image[1])
|
||||
if isinstance(image, os.PathLike):
|
||||
return Path(image).read_bytes()
|
||||
if isinstance(image, _SeekableBinaryReader):
|
||||
position: Final = image.tell()
|
||||
image.seek(0)
|
||||
data: Final = image.read()
|
||||
image.seek(position)
|
||||
return data
|
||||
raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}")
|
||||
|
||||
|
||||
def image_to_data_url(image: object) -> str:
|
||||
if isinstance(image, str):
|
||||
return image
|
||||
image_bytes: Final = _read_image_bytes(image)
|
||||
mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
|
||||
return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
|
|||
|
||||
_RESPONSE_CONTENT_FIELD: Final = "content"
|
||||
|
||||
_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
|
||||
|
||||
_ResponseT: Final = TypeVar("_ResponseT")
|
||||
|
||||
|
||||
|
|
@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
get_last_user_message,
|
||||
)
|
||||
|
||||
query: Final = get_last_user_message(cast(list[AllMessageValues], messages))
|
||||
last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or ""
|
||||
query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip()
|
||||
if not query:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.interactions import (
|
||||
InteractionsAPIStreamingResponse,
|
||||
)
|
||||
|
|
@ -145,7 +146,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
|||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
self.stream_iterator = response.aiter_lines()
|
||||
self.stream_iterator = aiter_sse_lines(response.aiter_text())
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
|
@ -212,7 +213,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
|
|||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
self.stream_iterator = response.iter_lines()
|
||||
self.stream_iterator = iter_sse_lines(response.iter_text())
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"rpm",
|
||||
"itpm",
|
||||
"otpm",
|
||||
"tpd",
|
||||
"rpd",
|
||||
"use_xai_oauth",
|
||||
"chatgpt_auth_file",
|
||||
"github_copilot_auth_type",
|
||||
|
|
|
|||
|
|
@ -3481,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
level=LogfireLevel.INFO.value,
|
||||
level=LogfireLevel.INFO,
|
||||
)
|
||||
|
||||
if callback == "lunary" and lunaryLogger is not None:
|
||||
|
|
@ -4442,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
level=LogfireLevel.ERROR.value,
|
||||
level=LogfireLevel.ERROR,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timezone, tzinfo
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypedDict, cast
|
||||
from typing import Final, Literal, Protocol, TypedDict, cast
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from typing_extensions import ReadOnly
|
||||
|
|
@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost(
|
|||
)
|
||||
|
||||
|
||||
class _ModelPricedImageCostCalculator(Protocol):
|
||||
def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ...
|
||||
|
||||
|
||||
def _model_priced_image_cost_calculator(
|
||||
custom_llm_provider: str | None, model: str
|
||||
) -> _ModelPricedImageCostCalculator | None:
|
||||
match custom_llm_provider:
|
||||
case litellm.LlmProviders.RECRAFT.value:
|
||||
from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator
|
||||
|
||||
return recraft_image_cost_calculator
|
||||
case litellm.LlmProviders.AIML.value:
|
||||
from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator
|
||||
|
||||
return aiml_image_cost_calculator
|
||||
case litellm.LlmProviders.COMETAPI.value:
|
||||
from litellm.llms.cometapi.image_generation.cost_calculator import (
|
||||
cost_calculator as cometapi_image_cost_calculator,
|
||||
)
|
||||
|
||||
return cometapi_image_cost_calculator
|
||||
case litellm.LlmProviders.RUNWAYML.value:
|
||||
from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator
|
||||
|
||||
return runwayml_image_cost_calculator
|
||||
case litellm.LlmProviders.TENCENT.value:
|
||||
from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator
|
||||
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
|
||||
|
||||
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
class CostCalculatorUtils:
|
||||
@staticmethod
|
||||
def call_type_has_image_response(call_type: str) -> bool:
|
||||
|
|
@ -2131,9 +2166,6 @@ class CostCalculatorUtils:
|
|||
from litellm.llms.gemini.image_generation.cost_calculator import (
|
||||
cost_calculator as gemini_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
|
||||
cost_calculator as vertex_ai_image_cost_calculator,
|
||||
)
|
||||
|
|
@ -2146,7 +2178,10 @@ class CostCalculatorUtils:
|
|||
)
|
||||
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
|
||||
pricing: Final = deployment_pricing(model_info)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
|
||||
|
||||
if model_priced_calculator is not None:
|
||||
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
|
||||
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
|
||||
if isinstance(completion_response, ImageResponse):
|
||||
return vertex_ai_image_cost_calculator(
|
||||
|
|
@ -2164,36 +2199,6 @@ class CostCalculatorUtils:
|
|||
optional_params=optional_params,
|
||||
)
|
||||
raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation")
|
||||
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
|
||||
return recraft_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.AIML.value:
|
||||
from litellm.llms.aiml.image_generation.cost_calculator import (
|
||||
cost_calculator as aiml_image_cost_calculator,
|
||||
)
|
||||
|
||||
return aiml_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
|
||||
from litellm.llms.cometapi.image_generation.cost_calculator import (
|
||||
cost_calculator as cometapi_image_cost_calculator,
|
||||
)
|
||||
|
||||
return cometapi_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
|
||||
if call_type in (
|
||||
CallTypes.image_edit.value,
|
||||
|
|
@ -2237,16 +2242,6 @@ class CostCalculatorUtils:
|
|||
optional_params=optional_params,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
|
||||
from litellm.llms.runwayml.cost_calculator import (
|
||||
cost_calculator as runwayml_image_cost_calculator,
|
||||
)
|
||||
|
||||
return runwayml_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif (
|
||||
custom_llm_provider == litellm.LlmProviders.OPENAI.value
|
||||
or custom_llm_provider == litellm.LlmProviders.AZURE.value
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.batch_utils import InMemoryFile
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.router_utils.batch_utils import InMemoryFile
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ def safe_json_structure(
|
|||
def _transform(key: str | None, value: str) -> str:
|
||||
return value if value_transform is None else value_transform(key, value)
|
||||
|
||||
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
|
||||
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
|
||||
# Check for maximum depth.
|
||||
if depth > max_depth:
|
||||
return "MaxDepthExceeded"
|
||||
|
|
@ -48,32 +48,32 @@ def safe_json_structure(
|
|||
if id(obj) in seen:
|
||||
return "CircularReference Detected"
|
||||
seen.add(id(obj))
|
||||
result: dict | list | tuple | set | str
|
||||
if isinstance(obj, dict):
|
||||
result = {}
|
||||
for k, v in obj.items():
|
||||
if isinstance(k, (str)):
|
||||
clean_k = k.replace("\x00", "") if "\x00" in k else k
|
||||
result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
|
||||
mapping_result: Final = {
|
||||
(clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
|
||||
for k, v in obj.items()
|
||||
if isinstance(k, str)
|
||||
}
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return mapping_result
|
||||
elif isinstance(obj, list):
|
||||
result = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return list_result
|
||||
elif isinstance(obj, tuple):
|
||||
result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
|
||||
tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return tuple_result
|
||||
elif isinstance(obj, set):
|
||||
result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
|
||||
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
set_result: Final = sorted(sortable_values)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return set_result
|
||||
elif isinstance(obj, BaseModel):
|
||||
dumped: Final = obj.model_dump()
|
||||
result = _serialize(dumped, seen, depth + 1, key)
|
||||
model_result: Final = _serialize(dumped, seen, depth + 1, key)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return model_result
|
||||
else:
|
||||
# Fall back to string conversion for non-serializable objects.
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1263,7 +1263,7 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
if _is_delta_empty:
|
||||
model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned
|
||||
model_response.choices[0].delta = Delta(content=self._close_open_think_block())
|
||||
# get any function call arguments
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
finish_reason=self.received_finish_reason
|
||||
|
|
@ -1292,21 +1292,26 @@ class CustomStreamWrapper:
|
|||
if self.merge_reasoning_content_in_choices is not True:
|
||||
return
|
||||
delta: Final = model_response.choices[0].delta
|
||||
reasoning_content: Final = getattr(delta, "reasoning_content", None)
|
||||
reasoning_content: Final = getattr(delta, "reasoning_content", None) or ""
|
||||
content: Final = delta.content or ""
|
||||
if reasoning_content:
|
||||
self._emitted_disqualifying_content = True
|
||||
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
|
||||
opening: Final = "" if self.sent_first_thinking_block else "<think>"
|
||||
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
|
||||
self.sent_first_thinking_block = True
|
||||
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
|
||||
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
|
||||
delta.content = "</think>" + content
|
||||
self.sent_last_thinking_block = True
|
||||
|
||||
ends_thinking: Final = bool(content or delta.tool_calls or delta.function_call)
|
||||
inside_think_block: Final = self.sent_first_thinking_block and not self.sent_last_thinking_block
|
||||
opening: Final = "<think>" if reasoning_content and not inside_think_block else ""
|
||||
closing: Final = "</think>" if ends_thinking and (reasoning_content or inside_think_block) else ""
|
||||
if hasattr(delta, "reasoning_content"):
|
||||
del delta.reasoning_content
|
||||
if not reasoning_content and not closing:
|
||||
return
|
||||
delta.content = opening + reasoning_content + closing + content
|
||||
self.sent_first_thinking_block = True
|
||||
self.sent_last_thinking_block = bool(closing)
|
||||
self._emitted_disqualifying_content = self._emitted_disqualifying_content or bool(reasoning_content)
|
||||
|
||||
def _close_open_think_block(self) -> str | None:
|
||||
if not self.sent_first_thinking_block or self.sent_last_thinking_block:
|
||||
return None
|
||||
self.sent_last_thinking_block = True
|
||||
return "</think>"
|
||||
|
||||
def _dispatch_provider_chunk(
|
||||
self,
|
||||
|
|
@ -1932,8 +1937,20 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].finish_reason == "stop" and self.tool_call
|
||||
): # don't overwrite for other - potential error finish reasons
|
||||
model_response.choices[0].finish_reason = "tool_calls"
|
||||
close_tag: Final = self._close_open_think_block()
|
||||
if close_tag is not None:
|
||||
model_response.choices[0].delta.content = close_tag
|
||||
return model_response
|
||||
|
||||
def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
return processed_chunk
|
||||
if processed_chunk.choices[0].delta.content is None:
|
||||
return None
|
||||
return processed_chunk.model_copy(
|
||||
update={"choices": [processed_chunk.choices[0].model_copy(update={"finish_reason": None})]}
|
||||
)
|
||||
|
||||
def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
|
||||
"""
|
||||
Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
|
||||
|
|
@ -2173,10 +2190,9 @@ class CustomStreamWrapper:
|
|||
else:
|
||||
self.sent_last_chunk = True
|
||||
processed_chunk: Final = self.finish_reason_handler()
|
||||
# The logged response is built from self.chunks; keep a finish_reason the provider sent on its
|
||||
# last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
self.chunks.append(processed_chunk)
|
||||
final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
|
||||
if final_chunk_for_logging is not None:
|
||||
self.chunks.append(final_chunk_for_logging)
|
||||
if self.stream_options is None:
|
||||
self._set_final_chunk_usage(processed_chunk)
|
||||
## LOGGING
|
||||
|
|
@ -2406,10 +2422,9 @@ class CustomStreamWrapper:
|
|||
else:
|
||||
self.sent_last_chunk = True
|
||||
processed_chunk: Final = self.finish_reason_handler()
|
||||
# The logged response is built from self.chunks; keep a finish_reason the provider sent on its
|
||||
# last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
self.chunks.append(processed_chunk)
|
||||
final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
|
||||
if final_chunk_for_logging is not None:
|
||||
self.chunks.append(final_chunk_for_logging)
|
||||
if self.stream_options is None:
|
||||
self._set_final_chunk_usage(processed_chunk)
|
||||
# see sync __next__'s sibling branch: deliberately do NOT restore
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
get_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.llms.anthropic import (
|
||||
ContentBlockDelta,
|
||||
ContentBlockStart,
|
||||
|
|
@ -233,7 +234,7 @@ async def make_call(
|
|||
raise AnthropicError(status_code=500, message=str(e))
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
json_mode=json_mode,
|
||||
speed=speed,
|
||||
|
|
@ -304,7 +305,7 @@ def make_sync_call(
|
|||
)
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(response.iter_text()),
|
||||
sync_stream=True,
|
||||
json_mode=json_mode,
|
||||
speed=speed,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.llms.anthropic.chat.handler import (
|
|||
)
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
Message,
|
||||
|
|
@ -269,7 +270,7 @@ def build_complete_streaming_response_legacy(
|
|||
|
||||
def extract_sse_data(event_str: str) -> dict | None:
|
||||
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
|
||||
for line in event_str.splitlines():
|
||||
for line in split_sse_lines(event_str):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("data:"):
|
||||
payload = stripped[len("data:") :].strip()
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
|
|||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
|
||||
from litellm.router_utils.baseline_request import (
|
||||
from litellm.router_strategy.auto_router.baseline_request import (
|
||||
BASELINE_PARAMETERS,
|
||||
capture_baseline_parameters,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.llms.azure_ai.agents.transformation import (
|
|||
AzureAIAgentsConfig,
|
||||
AzureAIAgentsError,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionAnnotationURLCitation,
|
||||
|
|
@ -632,7 +633,7 @@ class AzureAIAgentsHandler:
|
|||
|
||||
current_event = None
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
async for line in aiter_sse_lines(response.aiter_text()):
|
||||
line = line.strip()
|
||||
|
||||
if line.startswith("event:"):
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ class BaseModelResponseIterator:
|
|||
released (and a backend like vLLM aborts generation) when the stream
|
||||
is abandoned before its natural end.
|
||||
|
||||
``streaming_response`` is usually a bare ``aiter_lines()`` generator
|
||||
``streaming_response`` is usually a bare ``aiter_sse_lines()`` generator
|
||||
that holds no reference to the response, so the handler that owns the
|
||||
response attaches it here after construction."""
|
||||
if self.http_response is not None:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row ha
|
|||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
|
||||
|
||||
|
|
@ -24,7 +25,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]:
|
|||
|
||||
events: Final[list[dict]] = []
|
||||
for block in chunk.split("\n\n"):
|
||||
for line in block.splitlines():
|
||||
for line in split_sse_lines(block):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("data:"):
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Final
|
|||
import httpx
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
SANDBOX_MAX_OUTPUT_BYTES: Final = 10 * 1024 * 1024
|
||||
|
|
@ -99,7 +100,7 @@ class BaseSandboxConfig:
|
|||
async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
|
||||
lines: Final[list[str]] = []
|
||||
total = 0
|
||||
async for line in response.aiter_lines():
|
||||
async for line in aiter_sse_lines(response.aiter_text()):
|
||||
total += len(line.encode("utf-8"))
|
||||
if total > SANDBOX_MAX_OUTPUT_BYTES:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import time
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -34,6 +36,15 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _PegasusResponse(TypedDict):
|
||||
message: ReadOnly[NotRequired[str | None]]
|
||||
finishReason: ReadOnly[NotRequired[str | None]]
|
||||
|
||||
|
||||
_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
|
||||
|
||||
|
||||
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
||||
"""
|
||||
Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
|
||||
|
|
@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
ModelResponse with choices[0].message.content and finish_reason
|
||||
"""
|
||||
try:
|
||||
completion_response: Final = raw_response.json()
|
||||
completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise BedrockError(
|
||||
message=f"Error parsing response: {raw_response.text}, error: {e}",
|
||||
|
|
@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
message_content: Final = completion_response.get("message", "")
|
||||
|
||||
# Extract finish reason and map to LiteLLM format
|
||||
finish_reason_raw: Final = completion_response.get("finishReason", "stop")
|
||||
finish_reason_raw: Final = completion_response.get("finishReason") or "stop"
|
||||
finish_reason: Final = map_finish_reason(finish_reason_raw)
|
||||
|
||||
# Set the response content
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import copy
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
|||
|
||||
_GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _CohereGeneration(TypedDict):
|
||||
text: ReadOnly[str | None]
|
||||
finish_reason: ReadOnly[str | None]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _CohereInvokeResponse(TypedDict):
|
||||
text: NotRequired[ReadOnly[str | None]]
|
||||
generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _LlamaInvokeResponse(TypedDict):
|
||||
generation: ReadOnly[str | None]
|
||||
|
||||
|
||||
_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
|
||||
_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
|
||||
|
||||
_GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
||||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
|
@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
outputText: str | None = None
|
||||
try:
|
||||
if provider == "cohere":
|
||||
if "text" in completion_response:
|
||||
outputText = completion_response["text"]
|
||||
elif "generations" in completion_response:
|
||||
outputText = completion_response["generations"][0]["text"]
|
||||
cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
|
||||
if "text" in cohere_response:
|
||||
outputText = cohere_response["text"]
|
||||
elif "generations" in cohere_response:
|
||||
outputText = cohere_response["generations"][0]["text"]
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
completion_response["generations"][0]["finish_reason"]
|
||||
cohere_response["generations"][0]["finish_reason"] or "stop"
|
||||
)
|
||||
elif provider == "anthropic":
|
||||
return litellm.AmazonAnthropicClaudeConfig().transform_response(
|
||||
|
|
@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
elif provider == "ai21":
|
||||
outputText = completion_response.get("completions")[0].get("data").get("text")
|
||||
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
|
||||
outputText = completion_response["generation"]
|
||||
outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
|
||||
elif provider == "mistral":
|
||||
outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
|
||||
else: # amazon titan
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.llms.base_llm.files.transformation import (
|
|||
BaseFilesConfig,
|
||||
LiteLLMLoggingObj,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.bedrock import AwsAuthParams, BedrockBatchRecordKind
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -400,7 +401,7 @@ def _batch_input_line(line_number: int, line: str) -> _BatchInputLine:
|
|||
|
||||
|
||||
def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]:
|
||||
numbered_lines: Final = enumerate(file_content.splitlines(), start=1)
|
||||
numbered_lines: Final = enumerate(split_sse_lines(file_content), start=1)
|
||||
return ((line_number, line) for line_number, line in numbered_lines if line.strip())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.llms.bedrock.passthrough.guardrail_translation.structured_edits imp
|
|||
structured_edits,
|
||||
tool_edits,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy.guardrails.auto_router_compression import is_compression_guardrail
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
|
|
@ -947,7 +948,7 @@ def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]:
|
|||
def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
_JSON_OBJECT.validate_python(json.loads(line[len("data:") :]))
|
||||
for line in sse.decode().splitlines()
|
||||
for line in split_sse_lines(sse.decode())
|
||||
if line.startswith("data:")
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.types.realtime import (
|
|||
RealtimeResponseTypedDict,
|
||||
)
|
||||
|
||||
_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
|
@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
List of Bedrock format messages (JSON strings)
|
||||
"""
|
||||
try:
|
||||
json_message: Final[dict[str, JsonValue]] = json.loads(message)
|
||||
json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
|
||||
except json.JSONDecodeError:
|
||||
verbose_logger.warning("Invalid JSON message: %s", message[:200])
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.llms.base_llm.search.transformation import (
|
|||
)
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
# AgentCore web-search rejects queries longer than 200 characters
|
||||
|
|
@ -146,7 +147,9 @@ def _iter_sse_events(text: str) -> Iterator[Mapping[str, object]]:
|
|||
progress notifications before the JSON-RPC response.
|
||||
"""
|
||||
for chunk in _SSE_EVENT_SEPARATOR.split(text):
|
||||
payload = "\n".join(line[len("data:") :].lstrip() for line in chunk.splitlines() if line.startswith("data:"))
|
||||
payload = "\n".join(
|
||||
line[len("data:") :].lstrip() for line in split_sse_lines(chunk) if line.startswith("data:")
|
||||
)
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
safe_convert_created_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.llms.openai.common_utils import OpenAIError
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.responses.sse_output_recovery import (
|
||||
|
|
@ -199,7 +200,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
error_message = None
|
||||
streamed_output_items: Final[dict[int, dict[str, object]]] = {}
|
||||
text_only_output_items: Final[dict[int, dict[str, object]]] = {}
|
||||
for chunk in body_text.splitlines():
|
||||
for chunk in split_sse_lines(body_text):
|
||||
parsed_chunk = parse_sse_json_chunk(chunk)
|
||||
if parsed_chunk is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.utils import TextChoices
|
||||
from litellm.utils import CustomStreamWrapper, TextCompletionResponse
|
||||
|
||||
|
|
@ -109,7 +110,7 @@ async def make_call(
|
|||
if response.status_code != 200:
|
||||
raise TextCompletionCodestralError(status_code=response.status_code, message=response.text)
|
||||
|
||||
completion_stream: Final = response.aiter_lines()
|
||||
completion_stream: Final = aiter_sse_lines(response.aiter_text())
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
|
|
@ -339,13 +340,13 @@ class CodestralTextCompletion:
|
|||
response = litellm.module_level_client.post(
|
||||
completion_url,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
data=json.dumps({**data, "stream": True}),
|
||||
stream=stream,
|
||||
)
|
||||
_response: Final = CustomStreamWrapper(
|
||||
response.iter_lines(),
|
||||
iter_sse_lines(response.iter_text()),
|
||||
model,
|
||||
custom_llm_provider="codestral",
|
||||
custom_llm_provider="text-completion-codestral",
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return _response
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_httpx_client,
|
||||
get_ssl_configuration,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
|
||||
from litellm.types.llms.openai import FileTypes
|
||||
from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
|
||||
|
|
@ -466,7 +467,7 @@ class BaseLLMAIOHTTPHandler:
|
|||
)
|
||||
else:
|
||||
completion_stream = provider_config.get_model_response_iterator(
|
||||
streaming_response=response.iter_lines(), sync_stream=True
|
||||
streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_httpx_client,
|
||||
header_value,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.responses.streaming_iterator import (
|
||||
BaseResponsesAPIStreamingIterator,
|
||||
MockResponsesAPIStreamingIterator,
|
||||
|
|
@ -1086,7 +1087,7 @@ class BaseLLMHTTPHandler:
|
|||
completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
|
||||
else:
|
||||
completion_stream = provider_config.get_model_response_iterator_for_request(
|
||||
streaming_response=response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(response.iter_text()),
|
||||
sync_stream=True,
|
||||
json_mode=json_mode,
|
||||
optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),
|
||||
|
|
@ -1229,7 +1230,7 @@ class BaseLLMHTTPHandler:
|
|||
completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
|
||||
else:
|
||||
completion_stream = provider_config.get_model_response_iterator_for_request(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
json_mode=json_mode,
|
||||
optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),
|
||||
|
|
|
|||
74
litellm/llms/custom_httpx/sse_lines.py
Normal file
74
litellm/llms/custom_httpx/sse_lines.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
Split a decoded Server-Sent Events body into lines the way the SSE spec does.
|
||||
|
||||
httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028,
|
||||
U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a
|
||||
`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR
|
||||
(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream)
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
|
||||
from typing import Final
|
||||
|
||||
_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
|
||||
_SSE_LINE_WITH_BREAK: Final = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+")
|
||||
|
||||
|
||||
def _split_lines(text: str) -> Sequence[str]:
|
||||
if "\r" not in text:
|
||||
return text.split("\n")
|
||||
crlf_folded: Final = text.replace("\r\n", "\n")
|
||||
if "\r" not in crlf_folded:
|
||||
return crlf_folded.split("\n")
|
||||
return _SSE_LINE_BREAK.split(text)
|
||||
|
||||
|
||||
class _SSELineSplitter:
|
||||
def __init__(self) -> None:
|
||||
self._partial_line: io.StringIO = io.StringIO()
|
||||
self._after_cr: bool = False
|
||||
|
||||
def feed(self, text: str) -> tuple[str, ...]:
|
||||
if not text:
|
||||
return ()
|
||||
fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text
|
||||
self._after_cr = fresh.endswith("\r")
|
||||
*ended, tail = _split_lines(fresh)
|
||||
if not ended:
|
||||
_ = self._partial_line.write(tail)
|
||||
return ()
|
||||
first: Final = self._partial_line.getvalue() + ended[0]
|
||||
self._partial_line = io.StringIO()
|
||||
_ = self._partial_line.write(tail)
|
||||
return (first, *ended[1:])
|
||||
|
||||
def flush(self) -> tuple[str, ...]:
|
||||
rest: Final = self._partial_line.getvalue()
|
||||
self._partial_line = io.StringIO()
|
||||
return (rest,) if rest else ()
|
||||
|
||||
|
||||
def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]:
|
||||
splitter: Final = _SSELineSplitter()
|
||||
for text in text_chunks:
|
||||
yield from splitter.feed(text)
|
||||
yield from splitter.flush()
|
||||
|
||||
|
||||
def split_sse_lines(text: str) -> tuple[str, ...]:
|
||||
return tuple(iter_sse_lines((text,)))
|
||||
|
||||
|
||||
def split_sse_lines_keepends(text: str) -> tuple[str, ...]:
|
||||
return tuple(_SSE_LINE_WITH_BREAK.findall(text))
|
||||
|
||||
|
||||
async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]:
|
||||
splitter: Final = _SSELineSplitter()
|
||||
async for text in text_chunks:
|
||||
for line in splitter.feed(text):
|
||||
yield line
|
||||
for line in splitter.flush():
|
||||
yield line
|
||||
|
|
@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
|
|
@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
|
|||
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _DashScopeUsage(TypedDict, total=False):
|
||||
prompt_tokens: NotRequired[ReadOnly[int]]
|
||||
total_tokens: NotRequired[ReadOnly[int]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _DashScopeEmbeddingResponse(TypedDict, total=False):
|
||||
data: NotRequired[ReadOnly[list[object]]]
|
||||
model: NotRequired[ReadOnly[str | None]]
|
||||
usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
|
||||
id: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
|
||||
|
||||
|
||||
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
||||
"""
|
||||
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
|
||||
|
|
@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
|||
message=message,
|
||||
)
|
||||
|
||||
parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
|
||||
model_response.object = "list"
|
||||
model_response.data = response_json.get("data", [])
|
||||
model_response.model = response_json.get("model", model)
|
||||
model_response.data = parsed_response.get("data", [])
|
||||
model_response.model = parsed_response.get("model", model)
|
||||
|
||||
usage: Final = response_json.get("usage") or {}
|
||||
usage: Final = parsed_response.get("usage") or {}
|
||||
prompt_tokens: Final = usage.get("prompt_tokens", 0)
|
||||
total_tokens: Final = usage.get("total_tokens", prompt_tokens)
|
||||
setattr(
|
||||
|
|
@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
|||
),
|
||||
)
|
||||
|
||||
if "id" in response_json:
|
||||
setattr(model_response, "id", response_json["id"])
|
||||
if "id" in parsed_response:
|
||||
setattr(model_response, "id", parsed_response["id"])
|
||||
|
||||
return model_response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
|
|||
)
|
||||
|
||||
|
||||
_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class ModelResponseIterator:
|
||||
def __init__(self, streaming_response, sync_stream: bool):
|
||||
self.streaming_response = streaming_response
|
||||
|
|
@ -99,7 +104,7 @@ class ModelResponseIterator:
|
|||
self.response_iterator = self.streaming_response
|
||||
chunk: Final = self._next_sse_data()
|
||||
try:
|
||||
json_chunk: Final = json.loads(chunk)
|
||||
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
|
||||
return self.chunk_parser(chunk=json_chunk)
|
||||
except ValueError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -126,7 +131,7 @@ class ModelResponseIterator:
|
|||
if chunk == "[DONE]":
|
||||
raise StopAsyncIteration
|
||||
try:
|
||||
json_chunk: Final = json.loads(chunk)
|
||||
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
|
||||
return self.chunk_parser(chunk=json_chunk)
|
||||
except ValueError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ from typing import Final
|
|||
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import FileTypes
|
||||
|
||||
from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url
|
||||
from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig
|
||||
|
||||
FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth"
|
||||
SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size")
|
||||
|
|
@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
|
|||
)
|
||||
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
|
||||
"prompt": prompt,
|
||||
"image_url": to_data_url(next(iter(images))),
|
||||
"image_url": image_to_data_url(next(iter(images))),
|
||||
**provider_params,
|
||||
}
|
||||
return request_body, ()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
import base64
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import (
|
||||
map_gpt_image_quality,
|
||||
|
|
@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SeekableBinaryReader(Protocol):
|
||||
def tell(self) -> int: ...
|
||||
|
||||
def seek(self, offset: int) -> int: ...
|
||||
|
||||
def read(self) -> bytes: ...
|
||||
|
||||
|
||||
def _read_image_bytes(image: object) -> bytes:
|
||||
if isinstance(image, bytes):
|
||||
return image
|
||||
if isinstance(image, tuple):
|
||||
return _read_image_bytes(image[1])
|
||||
if isinstance(image, os.PathLike):
|
||||
return Path(image).read_bytes()
|
||||
if isinstance(image, _SeekableBinaryReader):
|
||||
position: Final = image.tell()
|
||||
image.seek(0)
|
||||
data: Final = image.read()
|
||||
image.seek(position)
|
||||
return data
|
||||
raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}")
|
||||
|
||||
|
||||
def to_data_url(image: object) -> str:
|
||||
if isinstance(image, str):
|
||||
return image
|
||||
image_bytes: Final = _read_image_bytes(image)
|
||||
mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
|
||||
return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
|
||||
|
||||
|
||||
def _first(value: object) -> object:
|
||||
return value[0] if isinstance(value, list) and value else value
|
||||
|
||||
|
|
@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig):
|
|||
raise ValueError("Fal AI image edit requires at least one input image")
|
||||
mask: Final = _first(image_edit_optional_request_params.get("mask"))
|
||||
mask_field: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
|
||||
MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({})
|
||||
)
|
||||
provider_params: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
|
||||
)
|
||||
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
|
||||
"prompt": prompt,
|
||||
"image_urls": tuple(to_data_url(img) for img in images),
|
||||
"image_urls": tuple(image_to_data_url(img) for img in images),
|
||||
**mask_field,
|
||||
**provider_params,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
|||
BaseTranslation,
|
||||
StreamTransformSink,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -121,7 +122,7 @@ def _parse_json_dict_or_none(payload: str) -> Mapping[str, object] | None:
|
|||
def _sse_payload_texts(sse_text: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
text
|
||||
for line in sse_text.splitlines()
|
||||
for line in split_sse_lines(sse_text)
|
||||
if line.startswith("data:")
|
||||
for payload in (line[len("data:") :].strip(),)
|
||||
if payload and payload != "[DONE]"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching import InMemoryCache
|
||||
|
|
@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
|
|||
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _GigaChatTokenResponse(TypedDict):
|
||||
tok: NotRequired[ReadOnly[str | None]]
|
||||
access_token: NotRequired[ReadOnly[str | None]]
|
||||
exp: NotRequired[ReadOnly[int | float | str | None]]
|
||||
expires_at: NotRequired[ReadOnly[int | float | str | None]]
|
||||
|
||||
|
||||
_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
|
||||
|
||||
|
||||
class GigaChatAuthError(BaseLLMException):
|
||||
"""GigaChat authentication error."""
|
||||
|
||||
|
|
@ -230,7 +243,7 @@ async def _request_token_async(
|
|||
|
||||
def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
|
||||
"""Parse OAuth token response."""
|
||||
data: Final = response.json()
|
||||
data: Final = _TOKEN_RESPONSE.validate_python(response.json())
|
||||
|
||||
# GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
|
||||
access_token: Final = data.get("tok") or data.get("access_token")
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
)
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
|
@ -357,7 +358,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
|
|||
api_key: str | None = None,
|
||||
) -> list[dict[str, str]]:
|
||||
streamed_response: Final = CustomStreamWrapper(
|
||||
completion_stream=response.iter_lines(),
|
||||
completion_stream=iter_sse_lines(response.iter_text()),
|
||||
model=model,
|
||||
custom_llm_provider="huggingface",
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
|
@ -36,12 +37,12 @@ class LangGraphSSEStreamIterator:
|
|||
|
||||
def __iter__(self):
|
||||
"""Initialize sync iteration."""
|
||||
self.line_iterator = self.response.iter_lines()
|
||||
self.line_iterator = iter_sse_lines(self.response.iter_text())
|
||||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
"""Initialize async iteration."""
|
||||
self.async_line_iterator = self.response.aiter_lines()
|
||||
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
|
||||
return self
|
||||
|
||||
def _parse_sse_line(self, line: str) -> ModelResponseStream | None:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
|
||||
from litellm.utils import EmbeddingResponse, ModelResponse, Usage
|
||||
|
||||
from ..common_utils import OobaboogaError
|
||||
|
|
@ -74,7 +75,7 @@ def completion(
|
|||
stream=optional_params["stream"] if "stream" in optional_params else False,
|
||||
)
|
||||
if "stream" in optional_params and optional_params["stream"] is True:
|
||||
return response.iter_lines()
|
||||
return iter_sse_lines(response.iter_text())
|
||||
else:
|
||||
return oobabooga_config.transform_response(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
"""
|
||||
|
||||
import ssl
|
||||
from typing import Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
from litellm._logging import redact_string, verbose_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
|
|
@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
|||
from ..common_utils import is_openai_backed_api_base
|
||||
from ..openai import OpenAIChatCompletion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import WebSocket
|
||||
|
||||
|
||||
class OpenAIRealtime(OpenAIChatCompletion):
|
||||
"""
|
||||
|
|
@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
async def async_realtime(
|
||||
self,
|
||||
model: str,
|
||||
websocket: Any,
|
||||
websocket: "WebSocket",
|
||||
logging_obj: LiteLLMLogging,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm import LlmProviders
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
|
|
@ -65,7 +66,9 @@ async def make_call(
|
|||
model_response: Final = _fake_streamed_model_response(response.json())
|
||||
completion_stream = MockResponseIterator(model_response=model_response)
|
||||
else:
|
||||
completion_stream = ModelResponseIterator(streaming_response=response.aiter_lines(), sync_stream=False)
|
||||
completion_stream = ModelResponseIterator(
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False
|
||||
)
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
|
|
@ -103,7 +106,9 @@ def make_sync_call(
|
|||
model_response: Final = _fake_streamed_model_response(response.json())
|
||||
completion_stream = MockResponseIterator(model_response=model_response)
|
||||
else:
|
||||
completion_stream = ModelResponseIterator(streaming_response=response.iter_lines(), sync_stream=True)
|
||||
completion_stream = ModelResponseIterator(
|
||||
streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
|
|
|
|||
|
|
@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
|
|||
"""
|
||||
|
||||
import base64
|
||||
import builtins
|
||||
import struct
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
|
|
@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
|
|||
_INT: Final = TypeAdapter(int)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _PerplexityEmbeddingResponse(TypedDict, total=False):
|
||||
model: ReadOnly[str | None]
|
||||
object: ReadOnly[Literal["list"]]
|
||||
data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
|
||||
usage: ReadOnly[builtins.object]
|
||||
|
||||
|
||||
_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
|
||||
|
||||
|
||||
class PerplexityEmbeddingError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
|
|||
litellm_params: dict = {},
|
||||
) -> EmbeddingResponse:
|
||||
try:
|
||||
raw_response_json: Final = raw_response.json()
|
||||
raw_response_json: Final[object] = raw_response.json()
|
||||
except Exception:
|
||||
raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
|
||||
model_response.model = raw_response_json.get("model", model)
|
||||
model_response.object = raw_response_json.get("object", "list")
|
||||
response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
|
||||
raw_response_json
|
||||
)
|
||||
|
||||
raw_data: Final = raw_response_json.get("data", [])
|
||||
model_response.model = response_data.get("model", model)
|
||||
model_response.object = response_data.get("object", "list")
|
||||
|
||||
raw_data: Final = response_data.get("data", [])
|
||||
decoded_data: Final[list[dict[str, object]]] = []
|
||||
for item in raw_data:
|
||||
decoded_item = dict(_JSON_OBJECT.validate_python(item))
|
||||
|
|
@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
|
|||
decoded_data.append(decoded_item)
|
||||
model_response.data = decoded_data
|
||||
|
||||
usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
|
||||
usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=_INT.validate_python(
|
||||
usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse
|
||||
|
||||
from ..common_utils import PredibaseError
|
||||
|
|
@ -34,7 +35,7 @@ async def make_call(
|
|||
if response.status_code != 200:
|
||||
raise PredibaseError(status_code=response.status_code, message=response.text)
|
||||
|
||||
completion_stream: Final = response.aiter_lines()
|
||||
completion_stream: Final = aiter_sse_lines(response.aiter_text())
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
|
|
@ -163,7 +164,7 @@ class PredibaseChatCompletion:
|
|||
timeout=timeout,
|
||||
)
|
||||
_response: Final = CustomStreamWrapper(
|
||||
response.iter_lines(),
|
||||
iter_sse_lines(response.iter_text()),
|
||||
model,
|
||||
custom_llm_provider="predibase",
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,49 @@
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
calculate_image_response_cost_from_usage,
|
||||
flat_image_cost,
|
||||
generic_cost_per_token,
|
||||
resolve_image_model_info,
|
||||
)
|
||||
from litellm.types.utils import ImageResponse, ModelInfo, Usage
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
||||
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")
|
||||
|
||||
|
||||
def _deployment_image_token_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
|
||||
if model_info is None or model_info.get("output_cost_per_image_token") is not None:
|
||||
return model_info
|
||||
output_cost_per_token: Final = model_info.get("output_cost_per_token")
|
||||
if output_cost_per_token is None:
|
||||
return model_info
|
||||
priced: Final[ModelInfo] = {**model_info, "output_cost_per_image_token": output_cost_per_token}
|
||||
return priced
|
||||
|
||||
|
||||
def _deployment_prices_per_image_only(model_info: ModelInfo | None) -> bool:
|
||||
return (
|
||||
model_info is not None
|
||||
and model_info.get("output_cost_per_image") is not None
|
||||
and model_info.get("output_cost_per_image_token") is None
|
||||
and model_info.get("output_cost_per_token") is None
|
||||
)
|
||||
|
||||
|
||||
def image_cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
|
||||
resolved_model_info: Final = resolve_image_model_info(
|
||||
model=model, custom_llm_provider="tencent", model_info=_deployment_image_token_pricing(model_info)
|
||||
)
|
||||
if _deployment_prices_per_image_only(model_info):
|
||||
return flat_image_cost(resolved_model_info, image_response)
|
||||
token_based_cost: Final = calculate_image_response_cost_from_usage(
|
||||
model=model,
|
||||
image_response=image_response,
|
||||
custom_llm_provider="tencent",
|
||||
model_info=resolved_model_info,
|
||||
)
|
||||
if token_based_cost is not None:
|
||||
return token_based_cost
|
||||
return flat_image_cost(resolved_model_info, image_response)
|
||||
|
|
|
|||
0
litellm/llms/tencent/image_edit/__init__.py
Normal file
0
litellm/llms/tencent/image_edit/__init__.py
Normal file
147
litellm/llms/tencent/image_edit/transformation.py
Normal file
147
litellm/llms/tencent/image_edit/transformation.py
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import override
|
||||
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.tencent.image_generation.transformation import (
|
||||
TOKENHUB_IMAGE_PARAMS,
|
||||
tokenhub_image_headers,
|
||||
tokenhub_image_params,
|
||||
tokenhub_image_response,
|
||||
tokenhub_image_url,
|
||||
)
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
_IMAGES: Final = TypeAdapter(tuple[object, ...])
|
||||
_FORM_BOOLEANS: Final = MappingProxyType({"true": True, "false": False})
|
||||
_OPENAI_PARAMS: Final = frozenset(TOKENHUB_IMAGE_PARAMS)
|
||||
|
||||
_FormEntries: TypeAlias = tuple[tuple[tuple[str, ...], object], ...]
|
||||
|
||||
|
||||
def _form_scalar(value: object) -> object:
|
||||
match value:
|
||||
case str() if value.lower() in _FORM_BOOLEANS:
|
||||
return _FORM_BOOLEANS[value.lower()]
|
||||
case str() if value.removeprefix("-").isdigit():
|
||||
return int(value)
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _form_path(key: str) -> tuple[str, ...]:
|
||||
root, bracket, rest = key.partition("[")
|
||||
if not bracket or not rest.endswith("]"):
|
||||
return (key,)
|
||||
return (root, *rest.removesuffix("]").split("]["))
|
||||
|
||||
|
||||
def _nested_form_value(entries: _FormEntries) -> object:
|
||||
match entries:
|
||||
case (((), value),):
|
||||
return _form_scalar(value)
|
||||
case _:
|
||||
return _nested_form_fields(tuple(entry for entry in entries if entry[0]))
|
||||
|
||||
|
||||
def _nested_form_fields(entries: _FormEntries) -> Mapping[str, object]:
|
||||
roots: Final = tuple(dict.fromkeys(path[0] for path, _ in entries))
|
||||
return {
|
||||
root: _nested_form_value(tuple((path[1:], value) for path, value in entries if path[0] == root))
|
||||
for root in roots
|
||||
}
|
||||
|
||||
|
||||
def _tokenhub_form_params(params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
tokenhub_entries: Final = tuple(
|
||||
(_form_path(key), value) for key, value in params.items() if key not in _OPENAI_PARAMS
|
||||
)
|
||||
return {
|
||||
**{key: value for key, value in params.items() if key in _OPENAI_PARAMS},
|
||||
**_nested_form_fields(tokenhub_entries),
|
||||
}
|
||||
|
||||
|
||||
class TencentImageEditConfig(BaseImageEditConfig):
|
||||
@override
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return list(TOKENHUB_IMAGE_PARAMS)
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: mirrors override contract
|
||||
return dict(tokenhub_image_params(image_edit_optional_params, model, drop_params))
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
return dict(tokenhub_image_headers(headers, api_key, model))
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> str:
|
||||
return tokenhub_image_url(api_base)
|
||||
|
||||
@override
|
||||
def use_multipart_form_data(self) -> bool:
|
||||
return False
|
||||
|
||||
@override
|
||||
def transform_image_edit_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str | None,
|
||||
image: object,
|
||||
image_edit_optional_request_params: Mapping[str, object],
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: Mapping[str, str],
|
||||
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
|
||||
images: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
|
||||
return {
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"images": tuple(image_to_data_url(img) for img in images if img is not None),
|
||||
**_tokenhub_form_params(image_edit_optional_request_params),
|
||||
}, ()
|
||||
|
||||
@override
|
||||
def transform_image_edit_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> ImageResponse:
|
||||
image_response: Final = tokenhub_image_response(raw_response)
|
||||
if image_response is None:
|
||||
raise BaseLLMException(
|
||||
status_code=raw_response.status_code,
|
||||
message=raw_response.text,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
return image_response
|
||||
0
litellm/llms/tencent/image_generation/__init__.py
Normal file
0
litellm/llms/tencent/image_generation/__init__.py
Normal file
200
litellm/llms/tencent/image_generation/transformation.py
Normal file
200
litellm/llms/tencent/image_generation/transformation.py
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import override
|
||||
|
||||
from litellm.exceptions import AuthenticationError, UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
OpenAIImageGenerationOptionalParams,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
ImageObject,
|
||||
ImageResponse,
|
||||
ImageUsage,
|
||||
ImageUsageInputTokensDetails,
|
||||
LlmProviders,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
TOKENHUB_API_BASE: Final = "https://tokenhub-intl.tencentcloudmaas.com/v1"
|
||||
TOKENHUB_IMAGE_PATH: Final = "/wand/si-image/generation"
|
||||
TOKENHUB_IMAGE_MODEL_PREFIX: Final = "seedream-"
|
||||
TOKENHUB_IMAGE_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
|
||||
"n",
|
||||
"size",
|
||||
"response_format",
|
||||
"output_format",
|
||||
"background",
|
||||
"user",
|
||||
)
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
class _TokenHubImage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
url: str | None = None
|
||||
b64_json: str | None = None
|
||||
size: str | None = None
|
||||
output_format: str | None = None
|
||||
|
||||
|
||||
class _TokenHubUsage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class _TokenHubImageResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
|
||||
created: int
|
||||
data: tuple[_TokenHubImage, ...]
|
||||
tokenhub_usage: _TokenHubUsage | None = None
|
||||
|
||||
|
||||
def is_tokenhub_image_model(model: str) -> bool:
|
||||
return model.removeprefix(f"{LlmProviders.TENCENT.value}/").startswith(TOKENHUB_IMAGE_MODEL_PREFIX)
|
||||
|
||||
|
||||
def tokenhub_image_url(api_base: str | None) -> str:
|
||||
base: Final = api_base or get_secret_str("TENCENT_API_BASE") or TOKENHUB_API_BASE
|
||||
return join_api_base(ensure_api_route(base, "v1"), TOKENHUB_IMAGE_PATH)
|
||||
|
||||
|
||||
def tokenhub_image_headers(headers: Mapping[str, str], api_key: str | None, model: str) -> Mapping[str, str]:
|
||||
resolved_api_key: Final = api_key or get_secret_str("TENCENT_API_KEY")
|
||||
if not resolved_api_key:
|
||||
raise AuthenticationError(
|
||||
message="TENCENT_API_KEY is not set. Set it in the environment or pass api_key.",
|
||||
llm_provider="tencent",
|
||||
model=model,
|
||||
)
|
||||
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
|
||||
|
||||
|
||||
def tokenhub_image_params(params: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, object]:
|
||||
n: Final = params.get("n")
|
||||
if n is not None and n != 1 and not drop_params:
|
||||
raise UnsupportedParamsError(
|
||||
status_code=400,
|
||||
message=f"Tencent TokenHub returns one image per request, so n={n} is not supported for {model}. "
|
||||
+ "To send n=1 instead, set `litellm.drop_params = True`.",
|
||||
)
|
||||
return {
|
||||
key: value
|
||||
for key, value in params.items()
|
||||
if key != "n" and value is not None and not (key == "background" and value == "auto")
|
||||
}
|
||||
|
||||
|
||||
def tokenhub_image_response(raw_response: httpx.Response) -> ImageResponse | None:
|
||||
try:
|
||||
parsed: Final = _TokenHubImageResponse.model_validate_json(raw_response.content)
|
||||
except ValidationError:
|
||||
return None
|
||||
total_tokens: Final = parsed.tokenhub_usage.total_tokens if parsed.tokenhub_usage is not None else 0
|
||||
first_image: Final = parsed.data[0] if parsed.data else _TokenHubImage()
|
||||
return ImageResponse(
|
||||
created=parsed.created,
|
||||
data=[ImageObject(url=image.url, b64_json=image.b64_json) for image in parsed.data],
|
||||
size=first_image.size,
|
||||
output_format=first_image.output_format,
|
||||
usage=ImageUsage(
|
||||
input_tokens=0,
|
||||
input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0),
|
||||
output_tokens=total_tokens,
|
||||
total_tokens=total_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TencentImageGenerationConfig(BaseImageGenerationConfig):
|
||||
@override
|
||||
def get_supported_openai_params(
|
||||
self, model: str
|
||||
) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: mirrors override contract
|
||||
return list(TOKENHUB_IMAGE_PARAMS)
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: Mapping[str, object],
|
||||
optional_params: Mapping[str, object],
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: mirrors override contract
|
||||
return {**optional_params, **tokenhub_image_params(non_default_params, model, drop_params)}
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
return tokenhub_image_url(api_base)
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
return dict(tokenhub_image_headers(headers, api_key, model))
|
||||
|
||||
@override
|
||||
def transform_image_generation_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
headers: Mapping[str, str],
|
||||
) -> dict[str, object]: # mutable-ok: mirrors override contract
|
||||
extra_body: Final = _JSON_OBJECT.validate_python(optional_params.get("extra_body") or {})
|
||||
params: Final = {key: value for key, value in optional_params.items() if key != "extra_body"}
|
||||
return {"model": model, "prompt": prompt, **params, **extra_body}
|
||||
|
||||
@override
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: Mapping[str, object],
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
encoding: object,
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
image_response: Final = tokenhub_image_response(raw_response)
|
||||
if image_response is None:
|
||||
raise BaseLLMException(
|
||||
status_code=raw_response.status_code,
|
||||
message=raw_response.text,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
return image_response
|
||||
|
|
@ -22,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
convert_content_list_to_str,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.llms.vertex_ai.agent_engine.sse_iterator import (
|
||||
VertexAgentEngineResponseIterator,
|
||||
)
|
||||
|
|
@ -357,7 +358,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase):
|
|||
) -> VertexAgentEngineResponseIterator:
|
||||
"""Return a streaming iterator for SSE responses."""
|
||||
return VertexAgentEngineResponseIterator(
|
||||
streaming_response=raw_response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(raw_response.iter_text()),
|
||||
sync_stream=True,
|
||||
)
|
||||
|
||||
|
|
@ -465,7 +466,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase):
|
|||
|
||||
# Create iterator for SSE stream (async)
|
||||
completion_stream: Final = VertexAgentEngineResponseIterator(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
get_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
||||
|
|
@ -3241,7 +3242,7 @@ async def make_call(
|
|||
raise
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
logging_obj=logging_obj,
|
||||
response_headers=response.headers,
|
||||
|
|
@ -3312,7 +3313,7 @@ def make_sync_call(
|
|||
raise
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(response.iter_text()),
|
||||
sync_stream=True,
|
||||
logging_obj=logging_obj,
|
||||
response_headers=response.headers,
|
||||
|
|
@ -4190,7 +4191,7 @@ class ModelResponseIterator:
|
|||
log_failed_hop(e)
|
||||
return self._truncated_stream_end(token)
|
||||
self._start_hop(hop_response, token)
|
||||
self.streaming_response = hop_response.aiter_lines()
|
||||
self.streaming_response = aiter_sse_lines(hop_response.aiter_text())
|
||||
self.async_response_iterator = self.streaming_response
|
||||
return None
|
||||
|
||||
|
|
@ -4204,7 +4205,7 @@ class ModelResponseIterator:
|
|||
log_failed_hop(e)
|
||||
return self._truncated_stream_end(token)
|
||||
self._start_hop(hop_response, token)
|
||||
self.streaming_response = hop_response.iter_lines()
|
||||
self.streaming_response = iter_sse_lines(hop_response.iter_text())
|
||||
self.response_iterator = self.streaming_response
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI
|
|||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -32,6 +33,17 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class _PredictionsEnvelope(TypedDict):
|
||||
predictions: NotRequired[ReadOnly[dict[str, object]]]
|
||||
|
||||
|
||||
_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope)
|
||||
_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter(
|
||||
_PredictionsEnvelopeType,
|
||||
config=ConfigDict(strict=True, extra="allow"),
|
||||
)
|
||||
|
||||
|
||||
def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
|
||||
try:
|
||||
return VertexGemmaContainerError.model_validate(predictions)
|
||||
|
|
@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
|||
|
||||
def _unwrap_predictions_response(
|
||||
self,
|
||||
response_json: dict[str, Any],
|
||||
response_json: _PredictionsEnvelope,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Unwrap the Vertex Gemma predictions format to OpenAI format.
|
||||
|
|
@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
|||
message=f"Request failed: {response.text}",
|
||||
)
|
||||
|
||||
response_json: Final = response.json()
|
||||
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
|
||||
|
||||
# Unwrap predictions to get OpenAI-compatible response
|
||||
openai_response: Final = self._unwrap_predictions_response(response_json)
|
||||
|
|
@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
|
|||
message=f"Request failed: {response.text}",
|
||||
)
|
||||
|
||||
response_json: Final = response.json()
|
||||
response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
|
||||
|
||||
# Unwrap predictions to get OpenAI-compatible response
|
||||
openai_response: Final = self._unwrap_predictions_response(response_json)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from types import UnionType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import fields as pyd_fields
|
||||
|
||||
import litellm
|
||||
|
|
@ -37,6 +38,8 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class _EventModelClass(Protocol):
|
||||
@property
|
||||
|
|
@ -420,7 +423,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
@staticmethod
|
||||
def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]:
|
||||
return raw_response.json()
|
||||
return _RESPONSE_BODY.validate_python(raw_response.json())
|
||||
|
||||
@staticmethod
|
||||
def _annotation_origin(annotation: object) -> object:
|
||||
|
|
|
|||
|
|
@ -28853,6 +28853,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.039,
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_image_token_batches": 1.5e-05,
|
||||
"output_cost_per_reasoning_token": 2.5e-06,
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token_batches": 1.25e-06,
|
||||
|
|
@ -28910,6 +28911,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.134,
|
||||
"output_cost_per_image_token": 0.00012,
|
||||
"output_cost_per_image_token_batches": 6e-05,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.8e-05,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
|
|
@ -28963,12 +28965,13 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.134,
|
||||
"output_cost_per_image_token": 0.00012,
|
||||
"output_cost_per_image_token_batches": 6e-05,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.8e-05,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/images/generations",
|
||||
|
|
@ -29014,6 +29017,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0672,
|
||||
"output_cost_per_image_token": 6e-05,
|
||||
"output_cost_per_image_token_batches": 3e-05,
|
||||
"output_cost_per_token": 3e-06,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"output_cost_per_token_flex": 1.5e-06,
|
||||
|
|
@ -29059,9 +29063,10 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0672,
|
||||
"output_cost_per_image_token": 6e-05,
|
||||
"output_cost_per_image_token_batches": 3e-05,
|
||||
"output_cost_per_token": 3e-06,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/images/generations",
|
||||
|
|
@ -29106,6 +29111,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0336,
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_image_token_batches": 1.5e-05,
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-07,
|
||||
"output_cost_per_token_flex": 7.5e-07,
|
||||
|
|
@ -29672,10 +29678,12 @@
|
|||
},
|
||||
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
|
||||
"input_cost_per_token_batches": 6.25e-07,
|
||||
"input_cost_per_token_flex": 6.25e-07,
|
||||
"input_cost_per_token_priority": 2.25e-06,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 2.7e-05,
|
||||
"output_cost_per_token_batches": 5e-06,
|
||||
"output_cost_per_token_flex": 5e-06,
|
||||
|
|
@ -29690,6 +29698,7 @@
|
|||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4e-06,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 7.2e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
|
|
@ -29701,6 +29710,7 @@
|
|||
"output_cost_per_image": 0.00012,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.8e-05,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 9e-06,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"output_cost_per_token_flex": 6e-06,
|
||||
|
|
@ -30229,6 +30239,7 @@
|
|||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4e-06,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 7.2e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
|
|
@ -30240,6 +30251,7 @@
|
|||
"output_cost_per_image": 0.00012,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.8e-05,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 9e-06,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"output_cost_per_token_flex": 6e-06,
|
||||
|
|
@ -51910,6 +51922,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.039,
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_image_token_batches": 1.5e-05,
|
||||
"output_cost_per_reasoning_token": 2.5e-06,
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token_batches": 1.25e-06,
|
||||
|
|
@ -51968,6 +51981,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.134,
|
||||
"output_cost_per_image_token": 0.00012,
|
||||
"output_cost_per_image_token_batches": 6e-05,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.8e-05,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
|
|
@ -52002,13 +52016,14 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.134,
|
||||
"output_cost_per_image_token": 0.00012,
|
||||
"output_cost_per_image_token_batches": 6e-05,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.8e-05,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image",
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/images/generations",
|
||||
|
|
@ -52034,6 +52049,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0672,
|
||||
"output_cost_per_image_token": 6e-05,
|
||||
"output_cost_per_image_token_batches": 3e-05,
|
||||
"output_cost_per_token": 3e-06,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"output_cost_per_token_flex": 1.5e-06,
|
||||
|
|
@ -52061,10 +52077,11 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0672,
|
||||
"output_cost_per_image_token": 6e-05,
|
||||
"output_cost_per_image_token_batches": 3e-05,
|
||||
"output_cost_per_token": 3e-06,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/images/generations",
|
||||
|
|
@ -52089,6 +52106,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0336,
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_image_token_batches": 1.5e-05,
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-07,
|
||||
"output_cost_per_token_flex": 7.5e-07,
|
||||
|
|
@ -61986,6 +62004,39 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"tencent/seedream-image-v5.0-flash": {
|
||||
"litellm_provider": "tencent",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.018,
|
||||
"output_cost_per_image_token": 1.6e-06,
|
||||
"source": "https://cloud.tencent.com/document/product/1823/130055",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"tencent/seedream-image-v5.0-lite": {
|
||||
"litellm_provider": "tencent",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.035,
|
||||
"output_cost_per_image_token": 1.6e-06,
|
||||
"source": "https://www.tencentcloud.com/document/product/1300/78937",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"tencent/seedream-image-v5.0-pro": {
|
||||
"litellm_provider": "tencent",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.09,
|
||||
"output_cost_per_image_token": 1.6e-06,
|
||||
"source": "https://www.tencentcloud.com/document/product/1300/78937",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"cognition/swe-1.6": {
|
||||
"input_cost_per_token": 5e-07,
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
|
|
@ -74367,6 +74418,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0336,
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_image_token_batches": 1.5e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
|
|
@ -74412,6 +74464,7 @@
|
|||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.0336,
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_image_token_batches": 1.5e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
|
|
|
|||
|
|
@ -3793,8 +3793,8 @@
|
|||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"image_generations": true,
|
||||
"image_edits": true,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
|
||||
from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.a2a.version_convert import (
|
||||
A2AVersion,
|
||||
|
|
@ -320,7 +321,7 @@ async def _a2a_sse_event_source(
|
|||
"error": {"code": -32603, "message": resp.reason_phrase},
|
||||
}
|
||||
return
|
||||
async for line in resp.aiter_lines():
|
||||
async for line in aiter_sse_lines(resp.aiter_text()):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("data:"):
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ from typing import Final
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends
|
||||
|
||||
_MESSAGE_START_EVENT: Final = "message_start"
|
||||
_MESSAGE_START_MARKER: Final = b"message_start"
|
||||
_SSE_DATA_FIELD: Final = "data:"
|
||||
|
|
@ -52,7 +54,7 @@ def _restamped_data_line(line: str, requested_model: str) -> str | None:
|
|||
|
||||
|
||||
def _restamped_frame(frame: str, requested_model: str) -> str | None:
|
||||
lines: Final = frame.splitlines(keepends=True)
|
||||
lines: Final = split_sse_lines_keepends(frame)
|
||||
restamped: Final = tuple(_restamped_data_line(line, requested_model) for line in lines)
|
||||
if all(line is None for line in restamped):
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ from collections.abc import Mapping
|
|||
from typing import Any, Final
|
||||
|
||||
import requests
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from .exceptions import NotFoundError, UnauthorizedError
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class ModelsManagementClient:
|
||||
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
|
||||
|
|
@ -109,7 +112,7 @@ class ModelsManagementClient:
|
|||
try:
|
||||
response: Final = session.send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
|
|
@ -145,7 +148,7 @@ class ModelsManagementClient:
|
|||
try:
|
||||
response: Final = session.send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
|
|
@ -287,7 +290,7 @@ class ModelsManagementClient:
|
|||
try:
|
||||
response: Final = session.send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
|
|
|
|||
|
|
@ -139,8 +139,8 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
|||
)
|
||||
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
|
@ -266,6 +266,8 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
|
|||
AnthropicStreamModelRestamper,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
ROUTER_SCHEDULED_ROUTE_TYPES,
|
||||
SCHEDULER_PRIORITY_HEADER,
|
||||
LiteLLMProxyRequestSetup,
|
||||
add_litellm_data_to_request,
|
||||
get_metadata_variable_name,
|
||||
|
|
@ -2478,14 +2480,20 @@ class ProxyBaseLLMRequestProcessing:
|
|||
self.data["model"] = alias_target
|
||||
|
||||
aliased_model: Final = self.data.get("model")
|
||||
if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
|
||||
self.data.update(
|
||||
llm_router.priority_header_update(
|
||||
model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
|
||||
)
|
||||
)
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
|
||||
self.data["litellm_call_id"] = litellm_call_id
|
||||
bind_telemetry_call_id(litellm_call_id)
|
||||
DDSpanTagger.tag_call_id(self.data.get("litellm_call_id"))
|
||||
DDSpanTagger.tag_call_id(litellm_call_id)
|
||||
DDSpanTagger.tag_request(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
requested_model=self.data.get("model"),
|
||||
requested_model=aliased_model,
|
||||
)
|
||||
|
||||
### AUTO STREAM USAGE TRACKING ###
|
||||
|
|
|
|||
269
litellm/proxy/db/accounting_ledger.py
Normal file
269
litellm/proxy/db/accounting_ledger.py
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent
|
||||
from litellm.proxy.db.db_span import db_span
|
||||
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import Prisma
|
||||
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
AccountingEvent: TypeAlias = InboundUsageEvent | OutboundUsageEvent | CostEvent
|
||||
_JSON: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_INSTANT: Final[TypeAdapter[datetime]] = TypeAdapter(AwareDatetime)
|
||||
_STRINGS: Final = TypeAdapter(tuple[dict[str, str], ...])
|
||||
_TIMEOUT: Final = timedelta(seconds=10)
|
||||
|
||||
|
||||
class AccountingConflictError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AcceptedEvent:
|
||||
event_id: str
|
||||
kind: EventKind
|
||||
source_event_id: str | None
|
||||
occurred_at: datetime
|
||||
payload: str
|
||||
payload_hash: str
|
||||
targets: tuple[str, ...]
|
||||
schema_version: int
|
||||
version: int
|
||||
|
||||
|
||||
def _json(value: object) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
||||
|
||||
|
||||
def _prepare(
|
||||
event: AccountingEvent,
|
||||
targets: tuple[DeliveryTarget, ...],
|
||||
envelope: str,
|
||||
schema_version: int,
|
||||
version: int,
|
||||
) -> _AcceptedEvent:
|
||||
if type(schema_version) is not int or type(version) is not int or min(schema_version, version) < 1:
|
||||
raise ValueError("Accounting versions must be positive integers")
|
||||
validated: Final = type(event).model_validate_json(event.model_dump_json())
|
||||
payload: Final = _JSON.validate_json(envelope)
|
||||
if payload.get("event") != validated.model_dump(mode="json"):
|
||||
raise ValueError("Accounting envelope must contain the matching public event")
|
||||
destinations: Final = tuple(sorted(frozenset(DeliveryTarget(target).value for target in targets)))
|
||||
serialized: Final = _json(payload)
|
||||
identity: Final = _json((schema_version, version, destinations, payload))
|
||||
kind: Final = (
|
||||
EventKind.COST
|
||||
if isinstance(validated, CostEvent)
|
||||
else EventKind.INBOUND_USAGE
|
||||
if isinstance(validated, InboundUsageEvent)
|
||||
else EventKind.OUTBOUND_USAGE
|
||||
)
|
||||
return _AcceptedEvent(
|
||||
event_id=validated.event_id,
|
||||
kind=kind,
|
||||
source_event_id=validated.usage_event_id if isinstance(validated, CostEvent) else None,
|
||||
occurred_at=validated.occurred_at if isinstance(validated, CostEvent) else validated.ended_at,
|
||||
payload=serialized,
|
||||
payload_hash=hashlib.sha256(identity.encode()).hexdigest(),
|
||||
targets=destinations,
|
||||
schema_version=schema_version,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
_INSERT_EVENT: Final = """
|
||||
INSERT INTO "LiteLLM_AccountingEvent"
|
||||
(event_id, kind, source_event_id, schema_version, payload_hash, occurred_at, payload)
|
||||
VALUES ($1, $2::"AccountingEventKind", $3, $4::int, $5, $6::timestamp, $7::jsonb)
|
||||
ON CONFLICT (event_id) DO NOTHING RETURNING event_id
|
||||
"""
|
||||
_INSERT_DELIVERIES: Final = """
|
||||
INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
|
||||
SELECT $1, value::"AccountingDeliveryTarget", $3::int FROM jsonb_array_elements_text($2::jsonb)
|
||||
"""
|
||||
_LOCK_SOURCE: Final = """
|
||||
SELECT event_id FROM "LiteLLM_AccountingEvent"
|
||||
WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE
|
||||
"""
|
||||
_EXISTING_COST: Final = """
|
||||
SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_event_id = $1
|
||||
"""
|
||||
_LOCK_EVENT: Final = """
|
||||
SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
|
||||
"""
|
||||
_COMPLETED_FAMILY: Final = """
|
||||
root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
|
||||
AND root.recorded_at < $1::timestamp
|
||||
AND (root.kind = 'inbound_usage' OR EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_AccountingEvent" AS cost WHERE cost.source_event_id = root.event_id AND cost.kind = 'cost'
|
||||
))
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_AccountingEvent" AS member
|
||||
WHERE (member.event_id = root.event_id OR (member.kind = 'cost' AND member.source_event_id = root.event_id))
|
||||
AND (member.recorded_at >= $1::timestamp
|
||||
OR EXISTS (SELECT 1 FROM "LiteLLM_AccountingDelivery" AS delivery
|
||||
WHERE delivery.event_id = member.event_id AND delivery.state != 'applied'))
|
||||
)
|
||||
"""
|
||||
_RETENTION_CANDIDATES: Final = (
|
||||
'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE '
|
||||
+ _COMPLETED_FAMILY
|
||||
+ " ORDER BY root.recorded_at, root.event_id LIMIT $2::int FOR UPDATE OF root SKIP LOCKED"
|
||||
)
|
||||
_ELIGIBLE_GROUP: Final = (
|
||||
'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE root.event_id = $2 AND ' + _COMPLETED_FAMILY
|
||||
)
|
||||
_DELETE_DELIVERIES: Final = """
|
||||
DELETE FROM "LiteLLM_AccountingDelivery" WHERE event_id IN (
|
||||
SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
|
||||
)
|
||||
"""
|
||||
_DELETE_EVENTS: Final = """
|
||||
DELETE FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
|
||||
"""
|
||||
|
||||
|
||||
_Conflict: TypeAlias = Literal["missing_source", "competing_cost", "identity_conflict"]
|
||||
|
||||
|
||||
async def _accept(db: Prisma, record: _AcceptedEvent) -> Literal["inserted", "duplicate"] | _Conflict:
|
||||
if record.source_event_id is not None:
|
||||
source: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_SOURCE, record.source_event_id))
|
||||
if not source:
|
||||
return "missing_source"
|
||||
costs: Final = _STRINGS.validate_python(tuple(await db.query_raw(_EXISTING_COST, record.source_event_id)))
|
||||
if any(cost["event_id"] != record.event_id for cost in costs):
|
||||
return "competing_cost"
|
||||
inserted: Final = _STRINGS.validate_python(
|
||||
await db.query_raw(
|
||||
_INSERT_EVENT,
|
||||
record.event_id,
|
||||
record.kind.value,
|
||||
record.source_event_id,
|
||||
record.schema_version,
|
||||
record.payload_hash,
|
||||
record.occurred_at.astimezone(timezone.utc).isoformat(),
|
||||
record.payload,
|
||||
)
|
||||
)
|
||||
if not inserted:
|
||||
existing: Final = _STRINGS.validate_python(tuple(await db.query_raw(_LOCK_EVENT, record.event_id)))
|
||||
if not existing or existing[0]["payload_hash"] != record.payload_hash:
|
||||
return "identity_conflict"
|
||||
return "duplicate"
|
||||
await db.execute_raw(_INSERT_DELIVERIES, record.event_id, _json(record.targets), record.version)
|
||||
return "inserted"
|
||||
|
||||
|
||||
async def _delete_completed(db: Prisma, event_id: str, cutoff: str) -> int:
|
||||
if not _STRINGS.validate_python(await db.query_raw(_ELIGIBLE_GROUP, cutoff, event_id)):
|
||||
return 0
|
||||
await db.execute_raw(_DELETE_DELIVERIES, event_id)
|
||||
return await db.execute_raw(_DELETE_EVENTS, event_id)
|
||||
|
||||
|
||||
class AccountingLedger:
|
||||
def __init__(self, transaction: Callable[[], _Transaction]) -> None:
|
||||
self._transaction: Final = transaction
|
||||
|
||||
@classmethod
|
||||
def for_client(cls, client: PrismaClient) -> AccountingLedger:
|
||||
from prisma.client import TransactionManager
|
||||
|
||||
managers: Final = TypeAdapter(TransactionManager, config=ConfigDict(arbitrary_types_allowed=True))
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction() -> AsyncGenerator[Prisma]:
|
||||
primary: Final = writer_wrapper(client.db)
|
||||
manager: Final = managers.validate_python(primary.tx(timeout=_TIMEOUT))
|
||||
async with db_span("accounting_ledger", "LiteLLM_AccountingEvent"), manager as db:
|
||||
await db.execute_raw("SET LOCAL statement_timeout = 5000")
|
||||
await db.execute_raw("SET LOCAL lock_timeout = 1000")
|
||||
yield db
|
||||
|
||||
return cls(transaction)
|
||||
|
||||
async def insert_event(
|
||||
self,
|
||||
event: AccountingEvent,
|
||||
targets: tuple[DeliveryTarget, ...],
|
||||
*,
|
||||
envelope: str,
|
||||
schema_version: int = 1,
|
||||
version: int = 1,
|
||||
) -> Literal["inserted", "duplicate"]:
|
||||
"""Accept T5/T6's serialized envelope, containing its public event under `event`.
|
||||
|
||||
The immutable acceptance hash includes the target set and delivery version.
|
||||
Changing either for an accepted event conflicts. T7 must provide a separate
|
||||
add-target or new-version workflow that preserves receipts and prevents
|
||||
duplicate contributions; redelivery cannot silently create or reset work.
|
||||
|
||||
The Prisma adapter opens one interactive transaction per event, including
|
||||
two SET LOCAL statements and a source-row lock for costs. I/O is async,
|
||||
but awaiting this method inline adds commit latency per event, potentially
|
||||
several times per request. T5 owns bounded asynchronous capture and an
|
||||
explicit batching strategy; looping over this method does not batch commits.
|
||||
Success acknowledges commit, independently of HTTP response acknowledgement.
|
||||
"""
|
||||
record: Final = _prepare(event, targets, envelope, schema_version, version)
|
||||
async with self._transaction() as db:
|
||||
result: Final = await _accept(db, record)
|
||||
match result:
|
||||
case "inserted" | "duplicate":
|
||||
return result
|
||||
case "missing_source":
|
||||
raise AccountingConflictError("Cost source must be a durably accepted outbound usage event")
|
||||
case "competing_cost":
|
||||
raise AccountingConflictError("Usage event already has an authoritative cost")
|
||||
case "identity_conflict":
|
||||
raise AccountingConflictError("Event identity conflicts with its accepted content or delivery set")
|
||||
case _:
|
||||
assert_never(result)
|
||||
|
||||
async def purge_completed(
|
||||
self,
|
||||
*,
|
||||
redelivery_before: datetime | None = None,
|
||||
recovery_before: datetime | None = None,
|
||||
limit: int = 100,
|
||||
) -> int:
|
||||
"""Delete at most `limit` completed families once both retention owners permit it.
|
||||
|
||||
T8 must supply a coherent recovery watermark including outstanding holds.
|
||||
Omitting either watermark disables deletion. Each usage row and its cost
|
||||
remain together so a retained charge never loses its source or vice versa.
|
||||
Applied receipts are deleted with the family, ending its retained dedup
|
||||
protection. The caller must guarantee events recorded before
|
||||
redelivery_before cannot be redelivered through supported paths.
|
||||
T8's recovery_before must also protect checkpoints,
|
||||
outstanding holds and unfinished settlements; age alone is insufficient.
|
||||
"""
|
||||
if type(limit) is not int or not 1 <= limit <= 1000:
|
||||
raise ValueError("Accounting retention batch size must be between 1 and 1000")
|
||||
if redelivery_before is None or recovery_before is None:
|
||||
return 0
|
||||
cutoff: Final = min(
|
||||
_INSTANT.validate_python(redelivery_before).astimezone(timezone.utc),
|
||||
_INSTANT.validate_python(recovery_before).astimezone(timezone.utc),
|
||||
).isoformat()
|
||||
async with self._transaction() as db:
|
||||
roots: Final = _STRINGS.validate_python(tuple(await db.query_raw(_RETENTION_CANDIDATES, cutoff, limit)))
|
||||
deleted: Final = tuple([await _delete_completed(db, root["event_id"], cutoff) for root in roots])
|
||||
return sum(deleted)
|
||||
|
|
@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
|
|||
|
||||
from litellm.types.llms.openai import OpenAIModerationResponse
|
||||
|
||||
return OpenAIModerationResponse(**response.json())
|
||||
return OpenAIModerationResponse.model_validate(response.json())
|
||||
|
||||
def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from litellm.constants import (
|
|||
HEALTH_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependency,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ from litellm.proxy.spend_tracking.savings import (
|
|||
_effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner
|
||||
_proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner
|
||||
)
|
||||
from litellm.router_strategy.auto_router.baseline_request import baseline_request
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_applied
|
||||
from litellm.router_utils.baseline_request import baseline_request
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
from litellm.types.utils import CallTypes, ModelInfo, Usage
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import Span
|
||||
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_base_model, get_bedrock_cross_region_inference_regions
|
||||
|
|
@ -30,7 +31,6 @@ from litellm.proxy.hooks.model_budget_shared_windows import (
|
|||
SharedWindows,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
|
||||
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload
|
||||
|
||||
|
|
|
|||
|
|
@ -118,6 +118,9 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
|
|||
|
||||
_SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority"
|
||||
ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"})
|
||||
|
||||
# W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
|
||||
# e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
|
||||
_TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
|
||||
|
|
@ -688,7 +691,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t
|
|||
|
||||
|
||||
def _strip_untrusted_request_header_controls(
|
||||
headers: Any,
|
||||
headers: object,
|
||||
*,
|
||||
allow_client_message_redaction_opt_out: bool = False,
|
||||
) -> None:
|
||||
|
|
@ -1692,7 +1695,7 @@ class LiteLLMProxyRequestSetup:
|
|||
|
||||
@staticmethod
|
||||
def add_internal_user_from_user_mapping(
|
||||
general_settings: dict | None,
|
||||
general_settings: Mapping[str, object] | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
headers: dict,
|
||||
) -> UserAPIKeyAuth:
|
||||
|
|
@ -1711,7 +1714,7 @@ class LiteLLMProxyRequestSetup:
|
|||
return user_api_key_dict
|
||||
|
||||
@staticmethod
|
||||
def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
|
||||
def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None:
|
||||
"""
|
||||
Get the user from the specified header if `general_settings.user_header_name` is set.
|
||||
"""
|
||||
|
|
@ -1732,7 +1735,9 @@ class LiteLLMProxyRequestSetup:
|
|||
return user
|
||||
|
||||
@staticmethod
|
||||
def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
|
||||
def get_openai_org_id_from_headers(
|
||||
headers: dict, general_settings: Mapping[str, object] | None = None
|
||||
) -> str | None:
|
||||
"""
|
||||
Get the OpenAI Org ID from the headers.
|
||||
"""
|
||||
|
|
@ -1823,7 +1828,7 @@ class LiteLLMProxyRequestSetup:
|
|||
headers: dict,
|
||||
request_data: Mapping[str, object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
general_settings: dict[str, object] | None = None,
|
||||
general_settings: Mapping[str, object] | None = None,
|
||||
) -> LitellmDataForBackendLLMCall:
|
||||
"""
|
||||
- Adds user from headers
|
||||
|
|
@ -3285,7 +3290,7 @@ def _per_user_credential_names_for_groups(
|
|||
GITHUB_COPILOT_AUTH_TYPE_KEY,
|
||||
GITHUB_COPILOT_PER_USER_AUTH_TYPE,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model
|
||||
|
||||
names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment
|
||||
for group in model_groups:
|
||||
|
|
|
|||
|
|
@ -60,16 +60,16 @@ from litellm.repositories.table_repositories import PrismaTableRepository
|
|||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependencyRole,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.request_models import (
|
||||
AutoRouterRoutingTestRequest,
|
||||
RequestComplexityRouterConfig,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
StrategyRouterDependencyRole,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
SHADOW_EVAL_TURN_VALVE,
|
||||
|
|
@ -231,9 +231,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa
|
|||
return _ShadowEvalAttemptRepository(prisma_client).table
|
||||
|
||||
|
||||
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]:
|
||||
async with db_span("auto_router_report_query", sql_relation(query)):
|
||||
return await prisma_client.db.query_raw(query, *args)
|
||||
return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args))
|
||||
|
||||
|
||||
async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int:
|
||||
|
|
@ -408,7 +411,7 @@ async def validate_complexity_router_config(
|
|||
auto_router_environment_reference_error(data.complexity_router_config, user_api_key_dict)
|
||||
)
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
validate_complexity_router_config_write,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,23 @@ from litellm.repositories.project_repository import ProjectRepository
|
|||
from litellm.repositories.table_repositories import ModelTableRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
carries_complexity_router_settings,
|
||||
count_capability_routers,
|
||||
gated_capabilities_of,
|
||||
is_complexity_router_model,
|
||||
validate_complexity_router_config_placement,
|
||||
validate_complexity_router_config_write,
|
||||
validate_strategy_router_model_write,
|
||||
)
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
tuning_quota_violation,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router import (
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
|
||||
ClassificationRubric,
|
||||
|
|
@ -144,20 +161,6 @@ from litellm.router_strategy.complexity_router import (
|
|||
normalize_classification_prompt,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import resolve_complexity_router_config_write
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
carries_complexity_router_settings,
|
||||
count_capability_routers,
|
||||
gated_capabilities_of,
|
||||
is_complexity_router_model,
|
||||
validate_complexity_router_config_placement,
|
||||
validate_complexity_router_config_write,
|
||||
validate_strategy_router_model_write,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.bedrock import AwsSessionTag
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U
|
|||
import inspect
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, get_args
|
||||
from typing import Final, get_args
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import Field
|
||||
|
|
@ -32,7 +32,7 @@ router: Final = APIRouter()
|
|||
|
||||
class RouterSettingsResponse(LiteLLMBaseModel):
|
||||
fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
|
||||
current_values: dict[str, Any] = Field(description="Current values of router settings")
|
||||
current_values: dict[str, object] = Field(description="Current values of router settings")
|
||||
routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
|
||||
routing_group_strategies: tuple[str, ...] = Field(
|
||||
description="Strategies supported when constructing a routing group"
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ from typing import Final
|
|||
from pydantic import Json, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
gated_capabilities_of,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
mutable_tuned_identities,
|
||||
tuning_quota_violation,
|
||||
|
|
|
|||
|
|
@ -34,13 +34,16 @@ from litellm.repositories.prisma_protocols import DatabaseClient
|
|||
from litellm.repositories.project_repository import ProjectRepository
|
||||
from litellm.repositories.table_repositories import TeamMembershipRepository
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import (
|
||||
ComplexityRouterConfigWrite,
|
||||
normalize_classifier_config_aliases,
|
||||
resolve_complexity_router_config_write,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
|
||||
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import Deployment, updateDeployment
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
|
|||
)
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import Prisma
|
||||
|
|
@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool:
|
|||
|
||||
|
||||
async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone:
|
||||
team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id})
|
||||
team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id})
|
||||
if team_row is None:
|
||||
return TeamGone(team_id=team_id)
|
||||
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
|
||||
|
|
|
|||
|
|
@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict:
|
|||
if decrypted_json is None:
|
||||
raise ValueError("Failed to decrypt state parameter")
|
||||
|
||||
state_data: Final = json.loads(decrypted_json)
|
||||
state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json))
|
||||
return state_data
|
||||
|
||||
|
||||
_MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class _ManagedAuthorizationCode(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.llms.anthropic.messages.stream_assembly import (
|
|||
split_sse_chunk_into_events,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import BEDROCK_INVOKE_TRACE_FIELD
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
|
||||
from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
|
||||
|
|
@ -61,7 +62,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
|
|||
|
||||
def _event_payload(event: str) -> Mapping[str, object] | None:
|
||||
data_lines: Final = tuple(
|
||||
line.strip()[len("data:") :].strip() for line in event.splitlines() if line.strip().startswith("data:")
|
||||
line.strip()[len("data:") :].strip() for line in split_sse_lines(event) if line.strip().startswith("data:")
|
||||
)
|
||||
if not data_lines or data_lines[0] in ("", "[DONE]"):
|
||||
return None
|
||||
|
|
@ -362,7 +363,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
) -> str | None:
|
||||
for raw in all_chunks:
|
||||
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
|
||||
for line in text.splitlines():
|
||||
for line in split_sse_lines(text):
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
try:
|
||||
|
|
@ -402,7 +403,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
"""
|
||||
for raw in reversed(all_chunks):
|
||||
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
|
||||
for line in reversed(text.splitlines()):
|
||||
for line in reversed(split_sse_lines(text)):
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
from litellm.types.passthrough_endpoints.tinyfish import (
|
||||
|
|
@ -81,7 +82,7 @@ def sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> bool:
|
|||
|
||||
|
||||
def run_id_from_sse_frames(frames: bytes) -> str | None:
|
||||
return _run_id_from_sse_chunks(frames.decode("utf-8", errors="replace").splitlines())
|
||||
return _run_id_from_sse_chunks(split_sse_lines(frames.decode("utf-8", errors="replace")))
|
||||
|
||||
|
||||
def resolve_tinyfish_agent_api_base() -> str:
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
|
|||
can_access_resource,
|
||||
resolve_resource_owner_id,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
|
||||
from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
|
||||
|
|
@ -830,7 +831,7 @@ _SSE_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue])
|
|||
|
||||
|
||||
def _first_streamed_response(frames: bytes) -> tuple[str, Mapping[str, JsonValue]] | None:
|
||||
for line in frames.decode("utf-8", errors="replace").splitlines():
|
||||
for line in split_sse_lines(frames.decode("utf-8", errors="replace")):
|
||||
if not line.startswith(_SSE_DATA_PREFIX):
|
||||
continue
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from pydantic import TypeAdapter
|
|||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.routing import BaseRoute, Route
|
||||
from starlette.websockets import WebSocketState
|
||||
from typing_extensions import ReadOnly
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import (
|
||||
ConnectionClosedError,
|
||||
|
|
@ -175,8 +176,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool:
|
|||
return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
|
||||
|
||||
|
||||
class _RegisteredPassThroughRoute(TypedDict):
|
||||
endpoint_id: ReadOnly[str]
|
||||
path: ReadOnly[str | None]
|
||||
type: ReadOnly[str]
|
||||
methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]]
|
||||
auth: ReadOnly[bool]
|
||||
passthrough_params: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
# Global registry to track registered pass-through routes and prevent memory leaks
|
||||
_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
|
||||
_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {}
|
||||
|
||||
|
||||
def get_response_body(response: httpx.Response) -> dict | None:
|
||||
|
|
@ -2267,7 +2277,7 @@ def create_pass_through_route(
|
|||
status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
|
||||
detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.",
|
||||
)
|
||||
target_params: Final = {
|
||||
target_params: Final[dict[str, object]] = {
|
||||
"target": target,
|
||||
"custom_headers": custom_headers,
|
||||
"forward_headers": _forward_headers,
|
||||
|
|
@ -2299,7 +2309,9 @@ def create_pass_through_route(
|
|||
|
||||
# Ensure custom_headers is a dict. Botocore returns a HeadersDict
|
||||
# for SigV4-prepared requests, which is a Mapping but not a dict.
|
||||
headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {}
|
||||
headers_dict: Final[dict[str, object]] = (
|
||||
{**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {}
|
||||
)
|
||||
|
||||
# Ensure query_params and custom_body are dicts or None
|
||||
final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {}
|
||||
|
|
@ -3132,9 +3144,7 @@ class SafeRouteAdder:
|
|||
return f"{path}/{{subpath:path}}"
|
||||
return None
|
||||
|
||||
def registered_route_methods(
|
||||
route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]],
|
||||
) -> tuple[tuple[str, str], ...]:
|
||||
def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]:
|
||||
starlette_path: Final = registered_route_path(route_info)
|
||||
methods: Final = route_info.get("methods")
|
||||
if starlette_path is None or not isinstance(methods, (list, tuple)):
|
||||
|
|
@ -3490,7 +3500,7 @@ class InitPassThroughEndpointHelpers:
|
|||
return (comparison_route, *forwarded)
|
||||
|
||||
@staticmethod
|
||||
def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None:
|
||||
def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None:
|
||||
"""Get passthrough params for a given route and optionally filter by HTTP method"""
|
||||
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
|
||||
for key in _registered_pass_through_routes:
|
||||
|
|
|
|||
|
|
@ -168,12 +168,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
capability_limit_violation,
|
||||
|
|
@ -181,12 +176,17 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
count_capability_routers,
|
||||
validate_complexity_router_config_placement,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
TUNING_BASELINE_PARAM_NAME,
|
||||
mutable_tuned_identities,
|
||||
snapshot_tuning_baselines,
|
||||
tuning_limit_violation,
|
||||
)
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.routing_groups import parse_routing_groups
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
|
|
@ -666,7 +666,11 @@ from litellm.proxy.list_api.common import (
|
|||
problem_response,
|
||||
request_validation_problem,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
SCHEDULER_PRIORITY_HEADER,
|
||||
LiteLLMProxyRequestSetup,
|
||||
add_litellm_data_to_request,
|
||||
)
|
||||
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
|
||||
rust_control_plane_router,
|
||||
)
|
||||
|
|
@ -18481,6 +18485,11 @@ async def async_queue_request(
|
|||
raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value})
|
||||
router: Final = llm_router
|
||||
|
||||
data.update(
|
||||
router.priority_header_update(
|
||||
model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
|
||||
)
|
||||
)
|
||||
queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data)
|
||||
if "stream" in data and data["stream"] is True: # use generate_responses to stream responses
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from starlette.types import Message
|
|||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
|
||||
|
|
@ -81,7 +82,7 @@ class _StreamsBody(Protocol):
|
|||
|
||||
|
||||
def _sse_frame_data(frame: str) -> str | None:
|
||||
return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None)
|
||||
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
|
||||
|
||||
|
||||
async def _never_receive() -> Message:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from datetime import datetime, timedelta
|
|||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel):
|
|||
last_owner: str | None = None
|
||||
|
||||
|
||||
_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
|
||||
_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
|
||||
_SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...])
|
||||
_DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...])
|
||||
|
|
@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout(
|
|||
await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL)
|
||||
for setting in planner_settings:
|
||||
await transaction.execute_raw(setting)
|
||||
return await transaction.query_raw(sql, *params)
|
||||
return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params))
|
||||
|
||||
|
||||
async def _reverse_hash_key_metadata(
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self._upstream_exhausted: bool = False
|
||||
self._response_id_primed: bool = False
|
||||
self._pending_tool_events: list[BaseLiteLLMOpenAIResponseObject] = []
|
||||
self._tool_calls_held_behind_text: tuple[object, ...] = ()
|
||||
self._tool_output_index_by_call_id: dict[str, int] = {}
|
||||
self._tool_args_by_call_id: dict[str, str] = {}
|
||||
self._tool_item_id_by_call_id: dict[str, str] = {} # mutable-ok: filled per call id as tool call events stream
|
||||
|
|
@ -388,6 +389,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return ""
|
||||
return self._tool_call_id_by_index.get(index, "")
|
||||
|
||||
def _hold_tool_calls_behind_text(self, tool_calls: object) -> None:
|
||||
if isinstance(tool_calls, list):
|
||||
self._tool_calls_held_behind_text = (*self._tool_calls_held_behind_text, *tool_calls)
|
||||
|
||||
def _release_held_tool_calls(self) -> tuple[object, ...]:
|
||||
held: Final = self._tool_calls_held_behind_text
|
||||
self._tool_calls_held_behind_text = ()
|
||||
return held
|
||||
|
||||
def _queue_tool_call_delta_events(self, tool_calls: object) -> None:
|
||||
"""
|
||||
Convert chat-completions streaming `tool_calls` deltas into Responses API streaming events.
|
||||
|
|
@ -1365,6 +1375,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if delta_content:
|
||||
if not self.sent_message_item_added_event:
|
||||
self._queue_message_item_added_events()
|
||||
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
|
||||
self._sequence_number += 1
|
||||
text_delta_event: Final = OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
|
||||
|
|
@ -1379,7 +1390,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# Priority 3: Handle tool call deltas (if any) -> queue events and emit them
|
||||
# For each tool call delta, we emit events one at a time to match OpenAI's streaming behavior
|
||||
if chunk.choices and hasattr(chunk.choices[0].delta, "tool_calls") and chunk.choices[0].delta.tool_calls:
|
||||
self._queue_tool_call_delta_events(chunk.choices[0].delta.tool_calls)
|
||||
self._queue_tool_call_delta_events([*self._release_held_tool_calls(), *chunk.choices[0].delta.tool_calls])
|
||||
# Return one pending tool event at a time
|
||||
if self._pending_tool_events:
|
||||
return self._pending_tool_events.pop(0)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,29 @@
|
|||
#
|
||||
# Thank you ! We ❤️ you! - Krrish & Ishaan
|
||||
|
||||
"""
|
||||
litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments.
|
||||
|
||||
Public surface. The router is moving behind a narrow interface; code outside
|
||||
the router should end up using a Router through two kinds of members only:
|
||||
|
||||
- endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages,
|
||||
the files, batches, assistants and vector store wrappers, ...), each a thin
|
||||
wrapper around one call;
|
||||
- entry points (being added) that replace direct access to internals: registry() for
|
||||
deployment lookups, settings() for configuration reads, health() for
|
||||
cooldowns, blocks and usage, select() to pick a deployment without calling
|
||||
it, apply() for runtime changes, observe() for routing events,
|
||||
register_plugin() for strategies and sub-routers, and retire() to release
|
||||
the router.
|
||||
|
||||
Everything else on the class is internal. The members outside code uses today
|
||||
are listed in tests/code_coverage_tests/router_surface_allowlist.json, and a CI
|
||||
check (router_surface_check.py next to it) fails on any new outside use, so the
|
||||
list only shrinks while callers move to the entry points. The same interface
|
||||
is documented for the Rust core in litellm-rust/crates/router-core.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
|
|
@ -85,6 +108,15 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
deployment_supported_endpoints,
|
||||
get_router_metadata_variable_name,
|
||||
is_batch_retrieve_call_type,
|
||||
replace_model_in_jsonl,
|
||||
should_replace_model_in_jsonl,
|
||||
without_deployments_that_cannot_batch,
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
|
|
@ -148,6 +180,15 @@ from litellm.llms.openai_like.model_info import (
|
|||
get_openai_compatible_model_info,
|
||||
)
|
||||
from litellm.responses.bridge_check import keep_registered_mode
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
AUTO_ROUTER_MODEL_PREFIX,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
claimed_capabilities,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
)
|
||||
from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
|
|
@ -187,24 +228,6 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
replace_complexity_router_headers,
|
||||
response_total_token_count,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
AUTO_ROUTER_MODEL_PREFIX,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
claimed_capabilities,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
)
|
||||
from litellm.router_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
deployment_supported_endpoints,
|
||||
get_router_metadata_variable_name,
|
||||
is_batch_retrieve_call_type,
|
||||
replace_model_in_jsonl,
|
||||
should_replace_model_in_jsonl,
|
||||
without_deployments_that_cannot_batch,
|
||||
)
|
||||
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
get_dynamic_litellm_params,
|
||||
|
|
@ -359,6 +382,7 @@ from litellm.router_utils.request_priority import (
|
|||
InvalidPriority,
|
||||
parse_default_priority,
|
||||
priority_error,
|
||||
priority_header_fields,
|
||||
request_drops_params,
|
||||
resolve_request_priority,
|
||||
unqueueable_priority_message,
|
||||
|
|
@ -1369,7 +1393,7 @@ class Router:
|
|||
},
|
||||
]
|
||||
|
||||
router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
|
||||
router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}])
|
||||
```
|
||||
"""
|
||||
|
||||
|
|
@ -1595,6 +1619,10 @@ class Router:
|
|||
|
||||
if default_fallbacks is not None or litellm.default_fallbacks is not None:
|
||||
_fallbacks = default_fallbacks or litellm.default_fallbacks
|
||||
if isinstance(_fallbacks, str):
|
||||
raise TypeError(
|
||||
f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']"
|
||||
)
|
||||
if self.fallbacks is not None:
|
||||
self.fallbacks.append({"*": _fallbacks})
|
||||
else:
|
||||
|
|
@ -2838,7 +2866,7 @@ class Router:
|
|||
self._initialize_core_endpoints()
|
||||
self._initialize_specialized_endpoints()
|
||||
|
||||
def validate_fallbacks(self, fallback_param: list | None):
|
||||
def validate_fallbacks(self, fallback_param: Sequence[Mapping[str, object]] | None):
|
||||
"""
|
||||
Validate the fallbacks parameter.
|
||||
"""
|
||||
|
|
@ -2851,6 +2879,13 @@ class Router:
|
|||
raise ValueError(
|
||||
f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."
|
||||
)
|
||||
((model_group, targets),) = fallback_dict.items()
|
||||
if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=[fallback_dict]):
|
||||
continue
|
||||
raise TypeError(
|
||||
f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. "
|
||||
f"Use {{'{model_group}': ['{targets}']}}"
|
||||
)
|
||||
|
||||
def _add_encrypted_content_affinity_check(self, enable_global_affinity: bool) -> None:
|
||||
def _move_before_deployment_affinity(
|
||||
|
|
@ -5411,14 +5446,22 @@ class Router:
|
|||
request_priority: Final = resolve_request_priority(
|
||||
requested=kwargs.get("priority"),
|
||||
default_priority=self.default_priority,
|
||||
drops_params=lambda: request_drops_params(
|
||||
kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
|
||||
),
|
||||
drops_params=lambda: self._drops_params(model=model, kwargs=kwargs),
|
||||
)
|
||||
if isinstance(request_priority, InvalidPriority):
|
||||
raise priority_error(model=model, message=request_priority.message)
|
||||
return request_priority, {key: value for key, value in kwargs.items() if key != "priority"}
|
||||
|
||||
def priority_header_update(
|
||||
self, model: str, kwargs: Mapping[str, object], header: str | None
|
||||
) -> Mapping[str, int | str]:
|
||||
return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs))
|
||||
|
||||
def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
|
||||
return request_drops_params(
|
||||
kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
|
||||
)
|
||||
|
||||
async def _schedule_factory(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -16251,7 +16294,7 @@ class Router:
|
|||
to the deployment that actually served the request. Every attempt therefore
|
||||
writes or clears, never just writes.
|
||||
"""
|
||||
from litellm.router_utils.baseline_request import capture_baseline_parameters
|
||||
from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
|
||||
phase_attributes(routing_decision_attributes(routing_decision))
|
||||
|
|
|
|||
|
|
@ -36,12 +36,12 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import RedisCache, RedisPipelineIncrementOperation, log_redis_failure
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
)
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
|
||||
from litellm.router_strategy.tag_based_routing import get_tags_from_request_kwargs
|
||||
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.router_utils.cooldown_callbacks import (
|
||||
get_prometheus_logger_from_callbacks,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
|
||||
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ from litellm import ModelResponse, token_counter, verbose_logger
|
|||
from litellm._internal_context import with_service_target
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue