mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into litellm_fireworks_minimax_m3_vision_pin
This commit is contained in:
commit
28c3a27265
313 changed files with 13028 additions and 2031 deletions
|
|
@ -1,6 +1,8 @@
|
|||
---
|
||||
name: database-access
|
||||
description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
|
||||
metadata:
|
||||
internal: true
|
||||
---
|
||||
|
||||
# Database access
|
||||
|
|
|
|||
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
|
|
@ -3,3 +3,4 @@
|
|||
/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
/.github/workflows/zizmor.yml @yuneng-berri
|
||||
|
|
|
|||
5
.github/workflows/cost-map-guard.yml
vendored
5
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -1,7 +1,7 @@
|
|||
name: Cost map guard
|
||||
|
||||
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
|
||||
pull_request_target:
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
|
|
@ -19,6 +19,7 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
id: revisions
|
||||
|
|
|
|||
10
.github/workflows/spend-logs-guard.yml
vendored
10
.github/workflows/spend-logs-guard.yml
vendored
|
|
@ -1,13 +1,9 @@
|
|||
name: Spend logs guard
|
||||
|
||||
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
|
||||
pull_request_target:
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
pull_request_review:
|
||||
types:
|
||||
- submitted
|
||||
- dismissed
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -23,7 +19,7 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
|
|
|
|||
3
.github/workflows/test-linting.yml
vendored
3
.github/workflows/test-linting.yml
vendored
|
|
@ -561,6 +561,9 @@ jobs:
|
|||
- name: router_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
|
||||
- name: router_surface_check
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
|
||||
|
||||
- name: test_chat_completion_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
|
||||
|
|
|
|||
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -245,6 +245,7 @@ jobs:
|
|||
tests/unit/llms/clarifai
|
||||
tests/unit/llms/claude_code
|
||||
tests/unit/llms/cloudflare
|
||||
tests/unit/llms/codestral
|
||||
tests/unit/llms/codex
|
||||
tests/unit/llms/cohere
|
||||
tests/unit/llms/cometapi
|
||||
|
|
|
|||
17
.github/workflows/zizmor.yml
vendored
17
.github/workflows/zizmor.yml
vendored
|
|
@ -27,10 +27,15 @@ jobs:
|
|||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install zizmor
|
||||
run: |
|
||||
curl -fsSL --retry 3 -o "$RUNNER_TEMP/zizmor.tar.gz" \
|
||||
https://github.com/zizmorcore/zizmor/releases/download/v1.31.1/zizmor-x86_64-unknown-linux-gnu.tar.gz
|
||||
echo "d185f726bdaf92df78273217dfdd24ebcd6b0ef48cf3ce776ff5762b6e138c99 $RUNNER_TEMP/zizmor.tar.gz" | sha256sum -c -
|
||||
tar xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$RUNNER_TEMP" zizmor
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
|
||||
with:
|
||||
version: "1.24.1"
|
||||
min-severity: medium
|
||||
advanced-security: false
|
||||
annotations: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
"$RUNNER_TEMP/zizmor" --no-ignores --no-config --min-severity=medium --format=github .
|
||||
|
|
|
|||
|
|
@ -2,9 +2,12 @@
|
|||
needs an approving review from the spend-log owner before it can merge.
|
||||
|
||||
CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
|
||||
so a path rule would put every migration on the owner. This gate reads the diff instead: when no
|
||||
added line in the watched files mentions the table it passes at once, otherwise it passes only once
|
||||
the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
|
||||
so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
|
||||
hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
|
||||
hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
|
||||
only once the owner's latest review on the pull request is an approval that still covers every spend-log line
|
||||
in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
|
||||
Reviews do not trigger the workflow, so the check is re-run by hand after approving.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -13,8 +16,10 @@ import argparse
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
|
@ -25,7 +30,10 @@ WATCHED: Final = (
|
|||
"litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
|
||||
)
|
||||
MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
|
||||
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
|
||||
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
|
||||
DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
|
||||
MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
|
||||
HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
|
||||
|
||||
|
||||
class Reviewer(BaseModel):
|
||||
|
|
@ -37,31 +45,77 @@ class Review(BaseModel):
|
|||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
state: str
|
||||
user: Reviewer
|
||||
commit_id: str | None = None
|
||||
|
||||
|
||||
_REVIEWS: Final = TypeAdapter(list[Review])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Block:
|
||||
"""Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
|
||||
|
||||
base: range
|
||||
head: range
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Hunk:
|
||||
removed: tuple[tuple[int, str], ...]
|
||||
added: tuple[tuple[int, str], ...]
|
||||
|
||||
|
||||
def watched(path: str) -> bool:
|
||||
return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
|
||||
|
||||
|
||||
def _added_lines(section: str) -> tuple[str, ...]:
|
||||
body: Final = section.split("\n@@", 1)
|
||||
return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
|
||||
def model_block(schema: str) -> range:
|
||||
"""1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
|
||||
lines: Final = schema.splitlines()
|
||||
starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
|
||||
if not starts:
|
||||
return range(0)
|
||||
ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
|
||||
return range(starts[0], (ends[0] if ends else len(lines)) + 1)
|
||||
|
||||
|
||||
def spend_log_hits(diff: str) -> tuple[str, ...]:
|
||||
"""`path: added line` for every added line in a watched file that names the table."""
|
||||
def _hunks(section: str) -> tuple[Hunk, ...]:
|
||||
headers: Final = tuple(HUNK.finditer(section))
|
||||
ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
|
||||
return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
|
||||
|
||||
|
||||
def _hunk(header: re.Match[str], body: str) -> Hunk:
|
||||
lines: Final = body.splitlines()
|
||||
return Hunk(
|
||||
removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
|
||||
added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
|
||||
)
|
||||
|
||||
|
||||
def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
|
||||
"""`path: +line` or `path: -line` for every changed line that touches the table.
|
||||
|
||||
A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
|
||||
merge base for removed ones), a migration line when it is added and names the table."""
|
||||
sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
|
||||
return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
|
||||
|
||||
|
||||
def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
|
||||
pairs: Final = tuple((_path_of(section), section) for section in sections)
|
||||
paths: Final = tuple((_path_of(section), section) for section in sections)
|
||||
return tuple(
|
||||
(path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
|
||||
) # comprehension-ok: path then its lines
|
||||
hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
|
||||
) # comprehension-ok: path then its hits
|
||||
|
||||
|
||||
def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
|
||||
if path not in blocks:
|
||||
return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
|
||||
block: Final = blocks[path]
|
||||
removed: Final = tuple(
|
||||
f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
|
||||
) # comprehension-ok: hunk then its lines
|
||||
added: Final = tuple(
|
||||
f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
|
||||
) # comprehension-ok: hunk then its lines
|
||||
return removed + added
|
||||
|
||||
|
||||
def _path_of(section: str) -> str:
|
||||
|
|
@ -70,11 +124,33 @@ def _path_of(section: str) -> str:
|
|||
|
||||
|
||||
def approved_by(reviews: Sequence[Review], login: str) -> bool:
|
||||
"""True when `login`'s most recent review on the pull request is an approval."""
|
||||
states: Final = tuple(review.state for review in reviews if review.user.login == login)
|
||||
"""True when `login`'s most recent review that changes GitHub's approval state is an approval.
|
||||
|
||||
COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too.
|
||||
"""
|
||||
own: Final = tuple(review.state for review in reviews if review.user.login == login)
|
||||
states: Final = tuple(state for state in own if state in DECIDING_STATES)
|
||||
return bool(states) and states[-1] == "APPROVED"
|
||||
|
||||
|
||||
def latest_approval(reviews: Sequence[Review], login: str) -> Review | None:
|
||||
"""`login`'s most recent APPROVED review on the pull request, the one GitHub keeps in force after pushes."""
|
||||
own: Final = tuple(review for review in reviews if review.user.login == login and review.state == "APPROVED")
|
||||
return own[-1] if own else None
|
||||
|
||||
|
||||
def approval_covers(
|
||||
approval: Review, head: str, hits: Sequence[str], hits_at: Callable[[str], Sequence[str] | None]
|
||||
) -> bool:
|
||||
"""True when the approval is on the head, or the head adds no spend-log line the approved commit did not."""
|
||||
if approval.commit_id is None:
|
||||
return False
|
||||
if approval.commit_id == head:
|
||||
return True
|
||||
reviewed: Final = hits_at(approval.commit_id)
|
||||
return reviewed is not None and set(hits) <= set(reviewed)
|
||||
|
||||
|
||||
def added_diff(base: str, head: str) -> str:
|
||||
result: Final = subprocess.run(
|
||||
("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
|
||||
|
|
@ -85,6 +161,27 @@ def added_diff(base: str, head: str) -> str:
|
|||
return result.stdout
|
||||
|
||||
|
||||
def file_at(revision: str, path: str) -> str:
|
||||
"""Contents of `path` at `revision`, empty when the file does not exist there."""
|
||||
result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
|
||||
return result.stdout if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
|
||||
return MappingProxyType(
|
||||
{path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
|
||||
)
|
||||
|
||||
|
||||
def fetched(commit: str) -> bool:
|
||||
result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
|
||||
return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
|
||||
|
||||
|
||||
class Arguments(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
base: str
|
||||
|
|
@ -100,18 +197,28 @@ def main(argv: Sequence[str]) -> int:
|
|||
parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
|
||||
parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
|
||||
args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
|
||||
hits: Final = spend_log_hits(added_diff(args.base, args.head))
|
||||
hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
|
||||
if not hits:
|
||||
sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
|
||||
sys.stdout.write(
|
||||
"No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
|
||||
"nothing to review.\n"
|
||||
)
|
||||
return 0
|
||||
if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
|
||||
reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
|
||||
approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
|
||||
if approval is not None and approval_covers(
|
||||
approval, args.head, hits, lambda commit: reviewed_hits(args.base, commit)
|
||||
):
|
||||
sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
|
||||
return 0
|
||||
listed: Final = "".join(f" {hit}\n" for hit in hits)
|
||||
sys.stdout.write(
|
||||
reason: Final = (
|
||||
f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
|
||||
f"{listed}The check re-runs when a review is submitted.\n"
|
||||
if approval is None
|
||||
else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, "
|
||||
f"a new approving review is needed:\n"
|
||||
)
|
||||
sys.stdout.write(f"{reason}{listed}After approving, re-run this job.\n")
|
||||
return 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import effective_skip_tool_message_for_guardrail
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails._content_utils import iter_tool_json_fields, map_tool_json_text, walk_user_text
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
|
@ -753,7 +754,12 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
|
|||
_masked_entity_count.set({})
|
||||
|
||||
# Covers multimodal list content + Responses-API input.
|
||||
inspected = walk_user_text(data, self.redact_text, self.redact_tool_payload) + self._redact_prompt(data)
|
||||
inspected = walk_user_text(
|
||||
data,
|
||||
self.redact_text,
|
||||
self.redact_tool_payload,
|
||||
skip_tool_message=effective_skip_tool_message_for_guardrail(self),
|
||||
) + self._redact_prompt(data)
|
||||
|
||||
if inspected == 0:
|
||||
# Image-only, empty-text, and unsupported payloads inspected
|
||||
|
|
|
|||
|
|
@ -343,11 +343,16 @@ def btree_max_entry_bytes(block_size: int) -> int:
|
|||
|
||||
|
||||
def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed":
|
||||
"""The bytes a value takes in the index entry: its length, or its stored size when
|
||||
Postgres compressed it, since `index_form_tuple` keeps the compressed bytes."""
|
||||
from psycopg import sql
|
||||
|
||||
if column.type_name in _OCTET_LENGTH_TYPES:
|
||||
return sql.SQL("octet_length({})").format(identifier)
|
||||
return sql.SQL("octet_length({}::text)").format(identifier)
|
||||
length: Final = (
|
||||
sql.SQL("octet_length({})").format(identifier)
|
||||
if column.type_name in _OCTET_LENGTH_TYPES
|
||||
else sql.SQL("octet_length({}::text)").format(identifier)
|
||||
)
|
||||
return sql.SQL("least({}, pg_column_size({}))").format(length, identifier)
|
||||
|
||||
|
||||
def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed":
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
---
|
||||
name: rust-error
|
||||
description: Define, wrap, convert, classify, report or test Rust error types in litellm-rust with thiserror. Use when adding an error type or variant, mapping one crate's error into another, turning an error into a gateway response or Python exception, deciding retry or fault from an error, or replacing call sites that build errors from strings
|
||||
metadata:
|
||||
internal: true
|
||||
---
|
||||
|
||||
# Rust errors
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
---
|
||||
name: rust-gateway
|
||||
description: Add or change an HTTP route, extractor, response envelope, stream or router mount in the litellm-rust gateway crates (crates/gateway and crates/gateway-*). Use when adding an endpoint, parsing a request body, choosing an error envelope for an API surface, relaying SSE, or writing a router-level test
|
||||
metadata:
|
||||
internal: true
|
||||
---
|
||||
|
||||
# Rust gateway
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
---
|
||||
name: rust-string-enums
|
||||
description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
|
||||
metadata:
|
||||
internal: true
|
||||
---
|
||||
|
||||
# Rust string enums
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
---
|
||||
name: rust-tracing
|
||||
description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery
|
||||
metadata:
|
||||
internal: true
|
||||
---
|
||||
|
||||
# Rust tracing
|
||||
|
|
|
|||
5
litellm-rust/Cargo.lock
generated
5
litellm-rust/Cargo.lock
generated
|
|
@ -4387,12 +4387,17 @@ name = "litellm-router"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-config",
|
||||
"litellm-core-utils",
|
||||
"litellm-inference",
|
||||
"litellm-inference-messages",
|
||||
"litellm-router-types",
|
||||
"rstest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-types"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,25 @@
|
|||
pub fn parse_str_bool(value: &str) -> Option<bool> {
|
||||
let token = value.trim_matches(|character: char| {
|
||||
fn python_strip(value: &str) -> &str {
|
||||
value.trim_matches(|character: char| {
|
||||
character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}')
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_str_bool(value: &str) -> Option<bool> {
|
||||
let token = python_strip(value);
|
||||
if token.eq_ignore_ascii_case("true") {
|
||||
return Some(true);
|
||||
}
|
||||
token.eq_ignore_ascii_case("false").then_some(false)
|
||||
}
|
||||
|
||||
pub fn parse_pydantic_bool(value: &str) -> Option<bool> {
|
||||
match python_strip(value).to_ascii_lowercase().as_str() {
|
||||
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
|
||||
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true.
|
||||
pub fn parse_redis_bool(value: &str) -> bool {
|
||||
value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
|
||||
|
|
@ -34,4 +46,27 @@ mod tests {
|
|||
) {
|
||||
assert_eq!(parse_str_bool(input), expected, "{input:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::yes(" YES ", Some(true))]
|
||||
#[case::on("On", Some(true))]
|
||||
#[case::one("1", Some(true))]
|
||||
#[case::t("t", Some(true))]
|
||||
#[case::y("y", Some(true))]
|
||||
#[case::control_whitespace_true("\u{1c}TRUE\u{1f}", Some(true))]
|
||||
#[case::no("No", Some(false))]
|
||||
#[case::off("OFF", Some(false))]
|
||||
#[case::zero("0", Some(false))]
|
||||
#[case::f("F", Some(false))]
|
||||
#[case::n("n", Some(false))]
|
||||
#[case::false_word("false", Some(false))]
|
||||
#[case::two("2", None)]
|
||||
#[case::empty("", None)]
|
||||
#[case::unknown("maybe", None)]
|
||||
fn boolean_words_follow_pydantic_lax_bool_parsing(
|
||||
#[case] input: &str,
|
||||
#[case] expected: Option<bool>,
|
||||
) {
|
||||
assert_eq!(parse_pydantic_bool(input), expected, "{input:?}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
|||
.map(|value| value.expose().to_owned())
|
||||
.or_else(|| std::env::var(name).ok())
|
||||
};
|
||||
let router = ModelRouter::from_config(config, &lookup);
|
||||
let settings = &config.litellm_settings;
|
||||
let http_settings = HttpSettings::from_layers([
|
||||
HttpSettingsLayer::from_environment(&lookup),
|
||||
|
|
@ -64,12 +65,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
|||
Arc::new(EnvironmentSecrets::python_compatible(client)),
|
||||
));
|
||||
let resources = CoreResources::new(pool);
|
||||
Ok(Arc::new(Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
secrets,
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)?))
|
||||
Ok(Arc::new(Gateway::new(resources, http, secrets, router)?))
|
||||
}
|
||||
|
||||
pub async fn build_redis(config: &Config) -> Result<RedisClients, Error> {
|
||||
|
|
|
|||
|
|
@ -141,9 +141,13 @@ async fn provider_call(
|
|||
&env_lookup,
|
||||
)?;
|
||||
|
||||
let body = config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
let body = config.request_body(
|
||||
config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
)?,
|
||||
&validated.headers,
|
||||
shaping.settings.drop_params,
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.request_headers(
|
||||
|
|
@ -514,6 +518,84 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
fn vertex_call_requesting(
|
||||
shaping: MessagesShaping,
|
||||
requested: &str,
|
||||
drop_params: bool,
|
||||
) -> MessagesCall {
|
||||
MessagesCall {
|
||||
body: body(
|
||||
json!({"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
),
|
||||
api_key: None,
|
||||
api_base: Some("https://vertex.test".into()),
|
||||
custom_llm_provider: Some("vertex_ai".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: Some(Map::from_iter([
|
||||
("authorization".to_string(), json!("Bearer vertex-token")),
|
||||
("anthropic-version".to_string(), json!(requested)),
|
||||
])),
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
drop_params,
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn vertex_project(name: &str) -> Option<String> {
|
||||
(name == "VERTEXAI_PROJECT").then(|| "version-project".to_string())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sdk_2("2026-10-21", false, "2026-10-21")]
|
||||
#[case::sdk_1("2023-06-01", false, "vertex-2023-10-16")]
|
||||
#[case::unknown_under_drop_params("latest", true, "vertex-2023-10-16")]
|
||||
#[tokio::test]
|
||||
async fn vertex_gets_the_requested_version_in_the_body_alone(
|
||||
shaping: MessagesShaping,
|
||||
#[case] requested: &str,
|
||||
#[case] drop_params: bool,
|
||||
#[case] body_version: &str,
|
||||
) {
|
||||
let call = provider_call_with_secrets(
|
||||
vertex_call_requesting(shaping, requested, drop_params),
|
||||
&vertex_project,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let version_headers: Vec<&str> = call
|
||||
.wire
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("anthropic-version"))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
(version_headers, &call.wire.body["anthropic_version"]),
|
||||
(Vec::<&str>::new(), &json!(body_version))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn vertex_refuses_an_unsupported_version_without_drop_params(shaping: MessagesShaping) {
|
||||
assert!(matches!(
|
||||
provider_call_with_secrets(
|
||||
vertex_call_requesting(shaping, "latest", false),
|
||||
&vertex_project
|
||||
)
|
||||
.await,
|
||||
Err(Error::Llm(litellm_llms::Error::InvalidRequest(
|
||||
litellm_llms::ErrorDetail::UnsupportedParameter { .. }
|
||||
)))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn missing_credentials_are_reported_before_the_request_is_transformed(
|
||||
|
|
|
|||
|
|
@ -77,6 +77,15 @@ pub trait BaseMessagesConfig: Sync {
|
|||
headers
|
||||
}
|
||||
|
||||
fn request_body(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
_headers: &Headers,
|
||||
_drop_params: bool,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
/// The JSON that goes on the wire, for a host whose body differs from the typed request:
|
||||
/// Python's configs `pop("model")` when the model is addressed by the URL.
|
||||
fn wire_body(&self, body: Value) -> Value {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# rules
|
||||
|
||||
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, puts the Vertex `anthropic_version` in the body, and authenticates with a Google access token
|
||||
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
|
||||
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
|
||||
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
|
||||
|
||||
# references
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ use litellm_auth_gcp::{
|
|||
VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
|
||||
get_vertex_ai_project_from_credentials, secret_names,
|
||||
};
|
||||
use litellm_http::request::has_header;
|
||||
use litellm_http::request::{has_header, header_value, without_headers};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, VERSION_HEADER},
|
||||
recognized::Recognized,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -36,6 +36,8 @@ use litellm_llms_types::providers::vertex_ai::{
|
|||
};
|
||||
|
||||
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
|
||||
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
|
||||
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
|
||||
|
||||
/// Claude on Vertex AI, Python's `VertexAIPartnerModelsAnthropicMessagesConfig`: the Anthropic
|
||||
/// payload addressed to a project and location, authenticated with a Google access token.
|
||||
|
|
@ -90,12 +92,6 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
.params
|
||||
.output_config
|
||||
.and_then(|config| sanitize_output_config(config, accepts_effort)),
|
||||
extra: request
|
||||
.params
|
||||
.extra
|
||||
.into_iter()
|
||||
.chain([("anthropic_version".to_string(), json!(ANTHROPIC_VERSION))])
|
||||
.collect(),
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
|
|
@ -133,7 +129,35 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
}
|
||||
|
||||
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
|
||||
merge_beta_headers(headers, vertex_feature_betas(request))
|
||||
merge_beta_headers(
|
||||
without_headers(headers, &[VERSION_HEADER]),
|
||||
vertex_feature_betas(request),
|
||||
)
|
||||
}
|
||||
|
||||
fn request_body(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
headers: &Headers,
|
||||
drop_params: bool,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
let version = vertex_version(
|
||||
&request.model,
|
||||
header_value(headers, VERSION_HEADER),
|
||||
drop_params,
|
||||
)?;
|
||||
Ok(MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
extra: request
|
||||
.params
|
||||
.extra
|
||||
.into_iter()
|
||||
.chain([("anthropic_version".to_string(), json!(version))])
|
||||
.collect(),
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
})
|
||||
}
|
||||
|
||||
/// Vertex addresses the model in the URL and rejects it in the body.
|
||||
|
|
@ -225,6 +249,26 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
|
|||
)
|
||||
}
|
||||
|
||||
fn vertex_version<'a>(
|
||||
model: &str,
|
||||
requested: Option<&'a str>,
|
||||
drop_params: bool,
|
||||
) -> Result<&'a str, Error> {
|
||||
match requested.map(str::trim).unwrap_or_default() {
|
||||
version if VERSIONS_SERVED_AS_VERTEX_DEFAULT.contains(&version) => Ok(ANTHROPIC_VERSION),
|
||||
version if VERSIONS_VERTEX_ACCEPTS_IN_BODY.contains(&version) => Ok(version),
|
||||
_ if drop_params => Ok(ANTHROPIC_VERSION),
|
||||
version => Err(Error::InvalidRequest(
|
||||
crate::ErrorDetail::UnsupportedParameter {
|
||||
model: model.into(),
|
||||
param: VERSION_HEADER.into(),
|
||||
value: version.into(),
|
||||
hint: String::new(),
|
||||
},
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
|
@ -471,6 +515,28 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sdk_spelling("anthropic-version")]
|
||||
#[case::any_case("Anthropic-Version")]
|
||||
fn a_forwarded_anthropic_version_never_reaches_vertex(#[case] name: &str) {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.request_headers(
|
||||
vec![
|
||||
(name.to_string(), "2026-10-21".to_string()),
|
||||
("x-request-id".to_string(), "req-1".to_string()),
|
||||
],
|
||||
&request,
|
||||
),
|
||||
vec![("x-request-id".to_string(), "req-1".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
fn transformed(value: Value, context: &MessagesTransformContext) -> Value {
|
||||
let request: MessagesRequest = serde_json::from_value(value).unwrap();
|
||||
serde_json::to_value(
|
||||
|
|
@ -482,16 +548,87 @@ mod tests {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
|
||||
let body = transformed(
|
||||
json!({
|
||||
"model": "claude-sonnet-4-5@20250929",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}),
|
||||
&MessagesTransformContext::default(),
|
||||
#[case::sdk_2(&[("anthropic-version", "2026-10-21")], false, "2026-10-21")]
|
||||
#[case::any_case(&[("Anthropic-Version", "2026-10-21")], false, "2026-10-21")]
|
||||
#[case::vertex_name(&[("anthropic-version", "vertex-2023-10-16")], false, "vertex-2023-10-16")]
|
||||
#[case::sdk_1(&[("anthropic-version", "2023-06-01")], false, "vertex-2023-10-16")]
|
||||
#[case::first_version(&[("anthropic-version", "2023-01-01")], false, "vertex-2023-10-16")]
|
||||
#[case::blank(&[("anthropic-version", " ")], false, "vertex-2023-10-16")]
|
||||
#[case::no_header(&[], false, "vertex-2023-10-16")]
|
||||
#[case::unknown_under_drop_params(&[("anthropic-version", "latest")], true, "vertex-2023-10-16")]
|
||||
fn the_requested_version_goes_in_the_body(
|
||||
#[case] forwarded: &[(&str, &str)],
|
||||
#[case] drop_params: bool,
|
||||
#[case] body_version: &str,
|
||||
) {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.unwrap();
|
||||
let headers: Headers = forwarded
|
||||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect();
|
||||
let body = serde_json::to_value(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.request_body(request, &headers, drop_params)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["anthropic_version"], json!(body_version));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unreleased("2025-01-01")]
|
||||
#[case::alias("latest")]
|
||||
fn an_unsupported_version_is_refused_without_drop_params(#[case] requested: &str) {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.unwrap();
|
||||
let headers: Headers = vec![("anthropic-version".to_string(), requested.to_string())];
|
||||
let Error::InvalidRequest(detail) = VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.request_body(request, &headers, false)
|
||||
.unwrap_err()
|
||||
else {
|
||||
panic!("expected an invalid request");
|
||||
};
|
||||
assert_eq!(
|
||||
detail.to_string(),
|
||||
format!(
|
||||
"claude-sonnet-4-5 does not support anthropic-version={requested}. \
|
||||
To drop unsupported params, set `litellm.drop_params = True`."
|
||||
)
|
||||
);
|
||||
assert_eq!(body["anthropic_version"], json!("vertex-2023-10-16"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5@20250929",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.unwrap();
|
||||
let body = serde_json::to_value(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.request_body(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.transform_anthropic_messages_request(
|
||||
request,
|
||||
&MessagesTransformContext::default(),
|
||||
)
|
||||
.unwrap(),
|
||||
&Vec::new(),
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["model"], json!("claude-sonnet-4-5@20250929"));
|
||||
let wire = VERTEX_ANTHROPIC_MESSAGES_CONFIG.wire_body(body);
|
||||
assert_eq!(wire.get("model"), None);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,36 @@
|
|||
//! The model catalog: per-model pricing, context limits and capabilities.
|
||||
//!
|
||||
//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
|
||||
//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
|
||||
//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
|
||||
//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
|
||||
//! in `litellm-router-core`, which reads this catalog through its dynamic config.
|
||||
//!
|
||||
//! # Structure
|
||||
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
|
||||
//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
|
||||
//! and keeps every entry in file order with its raw fields.
|
||||
//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
|
||||
//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
|
||||
//! modalities and modes in `capabilities`).
|
||||
//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
|
||||
//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
|
||||
//! failing the parse.
|
||||
//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
|
||||
//! families. They are data in the file, not entries.
|
||||
//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
|
||||
//! lost most of its models is refused), and `validate_registry` checks every entry against the
|
||||
//! checked-in contract.
|
||||
//!
|
||||
//! # Lookups
|
||||
//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
|
||||
//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
|
||||
//! prefixes are not stripped here; callers decide which keys to try.
|
||||
//!
|
||||
//! # Readers
|
||||
//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
|
||||
//! performs I/O; loading and refreshing the file is the caller's job.
|
||||
|
||||
mod capabilities;
|
||||
mod catalog;
|
||||
mod error;
|
||||
|
|
|
|||
9
litellm-rust/crates/router-core/Cargo.toml
Normal file
9
litellm-rust/crates/router-core/Cargo.toml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
|
||||
|
||||
[dependencies]
|
||||
63
litellm-rust/crates/router-core/src/lib.rs
Normal file
63
litellm-rust/crates/router-core/src/lib.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks
|
||||
//! and cooldowns around the attempts.
|
||||
//!
|
||||
//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in
|
||||
//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the
|
||||
//! same interface. Until then it holds this overview and nothing else, so the interface is written
|
||||
//! down in code before either side changes.
|
||||
//!
|
||||
//! # The public surface
|
||||
//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else:
|
||||
//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the
|
||||
//! files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over
|
||||
//! `call(op, request)`.
|
||||
//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`,
|
||||
//! `observe()`, `register_plugin()` and `retire()`.
|
||||
//!
|
||||
//! Every other member that outside code uses today has a named replacement among those. The old
|
||||
//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which
|
||||
//! implements them over the entry points and warns on use. A CI check
|
||||
//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside
|
||||
//! code touches; the list may only shrink.
|
||||
//!
|
||||
//! # Structure
|
||||
//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern
|
||||
//! and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`,
|
||||
//! `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and
|
||||
//! `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing
|
||||
//! and capabilities are the model catalog (`litellm-model-catalog`), which the router reads
|
||||
//! through its dynamic config.
|
||||
//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The
|
||||
//! router never reads process globals. Static config (model list, strategy, retry, timeout and
|
||||
//! cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config
|
||||
//! (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read
|
||||
//! once per call.
|
||||
//! - `Settings`: one typed, frozen view of the configuration a caller may read.
|
||||
//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and
|
||||
//! Rust workers see the same state. `report(id, outcome)` records outcomes the router did not
|
||||
//! see itself (a health check, an interceptor that marks a deployment failed).
|
||||
//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the
|
||||
//! configured strategy, without calling it.
|
||||
//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing
|
||||
//! a new registry version.
|
||||
//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
|
||||
//! policy is exhausted.
|
||||
//!
|
||||
//! # Execution
|
||||
//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
|
||||
//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
|
||||
//! started attempt is never replayed on a different executor.
|
||||
//!
|
||||
//! # Hosts
|
||||
//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
|
||||
//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
|
||||
//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
|
||||
//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
|
||||
//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
|
||||
//! serves them in the Rust server, where an operation without a Rust implementation makes the
|
||||
//! config fail at build time, never partway through a request.
|
||||
//!
|
||||
//! # Plugins
|
||||
//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
|
||||
//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
|
||||
//! the current request), never the router itself.
|
||||
|
|
@ -7,6 +7,7 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
litellm-config.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-inference.workspace = true
|
||||
litellm-inference-messages.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_model_list(&config.model_list)` maps configured public names to provider deployments. Programmatic callers can collect `(String, Deployment)` entries into a `Router`
|
||||
`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_config(&config, &env)` maps configured public names to provider deployments and reads `drop_params` from `litellm_settings` and each deployment, resolving an `os.environ/NAME` value through `env`. Programmatic callers can collect `(String, Deployment)` entries into a `Router`
|
||||
|
||||
Lookup is exact and returns `None` for an unknown name. This extraction preserves the gateway's existing behavior: the last entry wins when public names repeat. Multiple deployments per model group, routing strategies, retries, cooldowns, and fallbacks are not implemented yet
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,33 @@ mod deployment;
|
|||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_config::Model;
|
||||
use litellm_config::{Config, Spelled};
|
||||
use litellm_core_utils::{serde_compat::parse_pydantic_bool, settings::Lookup};
|
||||
use litellm_inference_messages::{MessagesSettings, MessagesShaping};
|
||||
|
||||
pub use deployment::Deployment;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Router(HashMap<String, Deployment>);
|
||||
|
||||
fn is_on(flag: Option<&Spelled<bool>>, env: &impl Lookup) -> bool {
|
||||
match flag {
|
||||
Some(Spelled::Value(on)) => *on,
|
||||
Some(Spelled::Text(text)) => match text.strip_prefix("os.environ/") {
|
||||
Some(name) => env
|
||||
.get(name)
|
||||
.is_some_and(|value| parse_pydantic_bool(&value) == Some(true)),
|
||||
None => parse_pydantic_bool(text) == Some(true),
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
impl Router {
|
||||
pub fn from_model_list(model_list: &[Model]) -> Self {
|
||||
model_list
|
||||
pub fn from_config(config: &Config, env: &impl Lookup) -> Self {
|
||||
let drops_params_globally = is_on(config.litellm_settings.drop_params.as_ref(), env);
|
||||
config
|
||||
.model_list
|
||||
.iter()
|
||||
.map(|model| {
|
||||
(
|
||||
|
|
@ -26,6 +43,14 @@ impl Router {
|
|||
api_base: model.litellm_params.api_base.clone(),
|
||||
custom_llm_provider: model.litellm_params.custom_llm_provider.clone(),
|
||||
litellm_params: model.litellm_params.clone(),
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
drop_params: drops_params_globally
|
||||
|| is_on(model.litellm_params.drop_params.as_ref(), env),
|
||||
..MessagesSettings::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..Deployment::default()
|
||||
},
|
||||
)
|
||||
|
|
@ -43,3 +68,60 @@ impl FromIterator<(String, Deployment)> for Router {
|
|||
Self(entries.into_iter().collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_config::Config;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::Router;
|
||||
|
||||
#[rstest]
|
||||
#[case::unset("", "", false)]
|
||||
#[case::deployment_flag("", "\n drop_params: true", true)]
|
||||
#[case::deployment_word("", "\n drop_params: \" True \"", true)]
|
||||
#[case::deployment_off("", "\n drop_params: false", false)]
|
||||
#[case::deployment_unknown_word("", "\n drop_params: maybe", false)]
|
||||
#[case::deployment_yes("", "\n drop_params: yes", true)]
|
||||
#[case::deployment_no("", "\n drop_params: \"no\"", false)]
|
||||
#[case::deployment_env_reference("", "\n drop_params: os.environ/DROP_PARAMS_FLAG", true)]
|
||||
#[case::deployment_unset_env_reference(
|
||||
"",
|
||||
"\n drop_params: os.environ/MISSING_FLAG",
|
||||
false
|
||||
)]
|
||||
#[case::global_env_reference(
|
||||
"litellm_settings:\n drop_params: os.environ/DROP_PARAMS_FLAG\n",
|
||||
"",
|
||||
true
|
||||
)]
|
||||
#[case::global_flag("litellm_settings:\n drop_params: true\n", "", true)]
|
||||
#[case::global_word("litellm_settings:\n drop_params: \"true\"\n", "", true)]
|
||||
#[case::global_off("litellm_settings:\n drop_params: false\n", "", false)]
|
||||
#[case::global_over_deployment_off(
|
||||
"litellm_settings:\n drop_params: true\n",
|
||||
"\n drop_params: false",
|
||||
true
|
||||
)]
|
||||
fn configured_drop_params_reach_the_deployment(
|
||||
#[case] settings: &str,
|
||||
#[case] deployment: &str,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let config = Config::from_yaml(&format!(
|
||||
"{settings}model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model{deployment}"
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
Router::from_config(&config, &|name: &str| (name == "DROP_PARAMS_FLAG")
|
||||
.then(|| " Yes ".to_owned()))
|
||||
.get("public-model")
|
||||
.unwrap()
|
||||
.shaping
|
||||
.settings
|
||||
.drop_params,
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ fn configuration_preserves_deployment_parameters(#[case] parameters: &str) {
|
|||
"model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model\n {parameters}"
|
||||
))
|
||||
.unwrap();
|
||||
let router = Router::from_model_list(&config.model_list);
|
||||
let router = Router::from_config(&config, &|_: &str| None);
|
||||
let deployment = router.get(&config.model_list[0].model_name).unwrap();
|
||||
let params = &config.model_list[0].litellm_params;
|
||||
|
||||
|
|
@ -44,7 +44,7 @@ fn configuration_types_the_provider_params_of_a_deployment() {
|
|||
azure_ad_token: ignored-by-every-group",
|
||||
)
|
||||
.unwrap();
|
||||
let router = Router::from_model_list(&config.model_list);
|
||||
let router = Router::from_config(&config, &|_: &str| None);
|
||||
let params = &router.get("public-model").unwrap().litellm_params;
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -74,7 +74,7 @@ fn lookup_uses_public_names(#[case] name: &str, #[case] expected: Option<&str>)
|
|||
model: provider/b",
|
||||
)
|
||||
.unwrap();
|
||||
let router = Router::from_model_list(&config.model_list);
|
||||
let router = Router::from_config(&config, &|_: &str| None);
|
||||
|
||||
assert_eq!(router.get(name).map(|entry| entry.model.as_str()), expected);
|
||||
}
|
||||
|
|
@ -84,7 +84,7 @@ fn empty_configuration_has_no_deployment() {
|
|||
let config = Config::from_yaml("model_list: []").unwrap();
|
||||
|
||||
assert!(
|
||||
Router::from_model_list(&config.model_list)
|
||||
Router::from_config(&config, &|_: &str| None)
|
||||
.get("")
|
||||
.is_none()
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1336,7 +1336,6 @@ all_embedding_models = (
|
|||
)
|
||||
|
||||
####### IMAGE GENERATION MODELS ###################
|
||||
openai_image_generation_models = ["dall-e-2", "dall-e-3"]
|
||||
|
||||
####### VIDEO GENERATION MODELS ###################
|
||||
openai_video_generation_models = ["sora-2"]
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.a2a_protocol.utils import (
|
|||
)
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
|
||||
RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33
|
||||
RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256
|
||||
|
|
@ -50,7 +51,7 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = (
|
|||
class _SSELineSource(Protocol):
|
||||
"""Minimal streaming-response surface used to read SSE lines."""
|
||||
|
||||
def aiter_lines(self) -> AsyncIterator[str]: ...
|
||||
def aiter_text(self) -> AsyncIterator[str]: ...
|
||||
|
||||
|
||||
def _filter_reserved_headers(
|
||||
|
|
@ -244,7 +245,7 @@ class BedrockAgentCoreA2ATransformation:
|
|||
Yields:
|
||||
Parsed JSON dicts from SSE data lines
|
||||
"""
|
||||
async for line in response.aiter_lines():
|
||||
async for line in aiter_sse_lines(response.aiter_text()):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
_IBM_CLOUD_IAM_URL: Final = "https://iam.cloud.ibm.com/identity/token"
|
||||
|
|
@ -74,7 +75,7 @@ class _WXORun(TypedDict, total=False):
|
|||
|
||||
|
||||
class _SSELineSource(Protocol):
|
||||
def aiter_lines(self) -> AsyncIterator[str]: ...
|
||||
def aiter_text(self) -> AsyncIterator[str]: ...
|
||||
|
||||
|
||||
class _WXOView(TypedDict, total=False):
|
||||
|
|
@ -231,7 +232,7 @@ class WatsonxOrchestrateHandler:
|
|||
async def _accumulate_wxo_sse_text(response: _SSELineSource) -> str:
|
||||
source: Final[_WXOView] = {"sse_source": response}
|
||||
accumulated_text = ""
|
||||
async for line in source["sse_source"].aiter_lines():
|
||||
async for line in aiter_sse_lines(source["sse_source"].aiter_text()):
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[5:].strip()
|
||||
|
|
|
|||
|
|
@ -274,7 +274,9 @@ class InMemoryCache(BaseCache):
|
|||
) -> list[float] | None:
|
||||
results: Final = []
|
||||
for increment in increment_list:
|
||||
result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs)
|
||||
result = await self.async_increment(
|
||||
increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs
|
||||
)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
|||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
CompletionTransformationBridge,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool
|
||||
from litellm.responses.sse_output_recovery import (
|
||||
parse_sse_json_chunk,
|
||||
|
|
@ -1351,7 +1352,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
recovered_output_items: Final[dict[int, dict[str, object]]] = {}
|
||||
recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
|
||||
|
||||
for chunk in raw_sse.splitlines():
|
||||
for chunk in split_sse_lines(raw_sse):
|
||||
parsed_chunk = parse_sse_json_chunk(chunk)
|
||||
if parsed_chunk is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -764,7 +764,6 @@ ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
|
|||
# Used for web search interception across providers
|
||||
LITELLM_WEB_SEARCH_TOOL_NAME: Final = "litellm_web_search"
|
||||
|
||||
DEFAULT_IMAGE_ENDPOINT_MODEL: Final = "dall-e-2"
|
||||
DEFAULT_VIDEO_ENDPOINT_MODEL: Final = "sora-2"
|
||||
|
||||
DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = get_env_int("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)
|
||||
|
|
|
|||
|
|
@ -1592,14 +1592,6 @@ def completion_cost(
|
|||
for part in ws_tier_parts
|
||||
)
|
||||
|
||||
if (
|
||||
(call_type == "aimage_generation" or call_type == "image_generation")
|
||||
and model is not None
|
||||
and isinstance(model, str)
|
||||
and len(model) == 0
|
||||
and custom_llm_provider == "azure"
|
||||
):
|
||||
model = "dall-e-2" # for dall-e-2, azure expects an empty model name
|
||||
# Handle Inputs to completion_cost
|
||||
prompt_tokens = 0
|
||||
prompt_characters: int | None = None
|
||||
|
|
@ -1674,7 +1666,8 @@ def completion_cost(
|
|||
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
)
|
||||
|
||||
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
model = candidate_model
|
||||
try:
|
||||
if verbose_logger.isEnabledFor(logging.DEBUG):
|
||||
verbose_logger.debug("selected model name for cost calculation: %s", model)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
|
|
@ -131,9 +132,9 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
|
|||
self.model = model
|
||||
self.generate_content_provider_config = generate_content_provider_config
|
||||
self.litellm_metadata = litellm_metadata
|
||||
# Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
|
||||
# Gemini streamGenerateContent uses SSE line framing; iter_sse_lines keeps
|
||||
# large inlineData payloads (e.g. image/jpeg) intact within one event.
|
||||
self.stream_iterator = response.iter_lines()
|
||||
self.stream_iterator = iter_sse_lines(response.iter_text())
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
|
@ -182,9 +183,9 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
|
|||
self.model = model
|
||||
self.generate_content_provider_config = generate_content_provider_config
|
||||
self.litellm_metadata = litellm_metadata
|
||||
# Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
|
||||
# Gemini streamGenerateContent uses SSE line framing; aiter_sse_lines keeps
|
||||
# large inlineData payloads (e.g. image/jpeg) intact within one event.
|
||||
self.stream_iterator = response.aiter_lines()
|
||||
self.stream_iterator = aiter_sse_lines(response.aiter_text())
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
import importlib
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from collections.abc import Collection, Coroutine, Mapping
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -16,7 +16,6 @@ import litellm
|
|||
|
||||
# client is imported from litellm as it's a decorator
|
||||
from litellm import client
|
||||
from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
|
||||
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
|
||||
from litellm.exceptions import LiteLLMUnknownProvider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
|
|
@ -61,6 +60,7 @@ from litellm.types.utils import (
|
|||
ProviderRoutingHints,
|
||||
)
|
||||
from litellm.utils import (
|
||||
EXTRA_BODY_ROUTING_KEYS,
|
||||
ImageResponse,
|
||||
ProviderConfigManager,
|
||||
get_llm_provider,
|
||||
|
|
@ -298,17 +298,14 @@ def image_generation(
|
|||
if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/"))
|
||||
else None
|
||||
)
|
||||
dynamic_api_key: str | None = None
|
||||
if model is not None or custom_llm_provider is not None:
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
litellm_params=chatgpt_params,
|
||||
)
|
||||
else:
|
||||
model = "dall-e-2"
|
||||
custom_llm_provider = "openai" # default to dall-e-2 on openai
|
||||
if model is None and custom_llm_provider is None:
|
||||
raise litellm.BadRequestError(message="model is required", model="", llm_provider="")
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
litellm_params=chatgpt_params,
|
||||
)
|
||||
model_response.hidden_params["model"] = model
|
||||
openai_params: Final = [
|
||||
"user",
|
||||
|
|
@ -466,7 +463,7 @@ def image_generation(
|
|||
litellm.LlmProviders.QWENCLOUD,
|
||||
litellm.LlmProviders.QWEN_AI_PLATFORM,
|
||||
litellm.LlmProviders.EDENAI,
|
||||
):
|
||||
) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None):
|
||||
if image_generation_config is None:
|
||||
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
|
||||
|
||||
|
|
@ -711,7 +708,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
|
|||
@client
|
||||
def image_variation(
|
||||
image: FileTypes,
|
||||
model: str = "dall-e-2", # set to dall-e-2 by default - like OpenAI.
|
||||
model: str | None = None,
|
||||
n: int = 1,
|
||||
response_format: Literal["url", "b64_json"] = "url",
|
||||
size: str | None = None,
|
||||
|
|
@ -727,7 +724,7 @@ def image_variation(
|
|||
litellm_params: Final = get_litellm_params(**kwargs)
|
||||
# get the custom llm provider
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
model=model or "",
|
||||
custom_llm_provider=litellm_params.get("custom_llm_provider", None),
|
||||
api_base=litellm_params.get("api_base", None),
|
||||
api_key=litellm_params.get("api_key", None),
|
||||
|
|
@ -746,7 +743,7 @@ def image_variation(
|
|||
response: ImageResponse | None = None
|
||||
|
||||
provider_config: Final = ProviderConfigManager.get_provider_model_info(
|
||||
model=model or "", # openai defaults to dall-e-2
|
||||
model=model,
|
||||
provider=llm_provider,
|
||||
)
|
||||
|
||||
|
|
@ -771,7 +768,7 @@ def image_variation(
|
|||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
model=model,
|
||||
model=model or None,
|
||||
image=image,
|
||||
timeout=litellm_params.get("timeout", None),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -789,7 +786,7 @@ def image_variation(
|
|||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
model=model,
|
||||
model=model or None,
|
||||
image=image,
|
||||
timeout=litellm_params.get("timeout", None) or DEFAULT_REQUEST_TIMEOUT,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -807,6 +804,32 @@ def image_variation(
|
|||
return response
|
||||
|
||||
|
||||
_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams))
|
||||
_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"})
|
||||
_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...])
|
||||
|
||||
|
||||
def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
|
||||
try:
|
||||
return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def _tokenhub_passthrough_edit_params(
|
||||
non_default_params: Mapping[str, object],
|
||||
extra_body: Mapping[str, object] | None,
|
||||
mapped_params: Collection[str],
|
||||
additional_drop_params: Collection[str] | None,
|
||||
) -> Mapping[str, object]:
|
||||
dropped: Final = frozenset(additional_drop_params or ())
|
||||
unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped
|
||||
return {
|
||||
**{key: value for key, value in non_default_params.items() if key not in unmapped_excluded},
|
||||
**{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS},
|
||||
}
|
||||
|
||||
|
||||
@client
|
||||
def image_edit(
|
||||
image: FileTypes | list[FileTypes] | None = None,
|
||||
|
|
@ -874,7 +897,7 @@ def image_edit(
|
|||
# get llm provider logic
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
model, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model or DEFAULT_IMAGE_ENDPOINT_MODEL,
|
||||
model=model or "",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
|
|
@ -935,22 +958,24 @@ def image_edit(
|
|||
raise ValueError(f"image edit is not supported for {custom_llm_provider}")
|
||||
|
||||
local_vars.update(kwargs)
|
||||
provider_supported_params: Final[frozenset[str]] = frozenset(
|
||||
image_edit_provider_config.get_supported_openai_params(model)
|
||||
)
|
||||
# Get ImageEditOptionalRequestParams with only valid parameters
|
||||
image_edit_optional_params: Final[ImageEditOptionalRequestParams] = (
|
||||
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
|
||||
local_vars,
|
||||
provider_supported_params=frozenset(
|
||||
image_edit_provider_config.get_supported_openai_params(model)
|
||||
).intersection(non_default_params),
|
||||
provider_supported_params=provider_supported_params.intersection(non_default_params),
|
||||
)
|
||||
)
|
||||
# Get optional parameters for the responses API
|
||||
additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params"))
|
||||
image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit(
|
||||
model=model,
|
||||
image_edit_provider_config=image_edit_provider_config,
|
||||
image_edit_optional_params=image_edit_optional_params,
|
||||
drop_params=kwargs.get("drop_params"),
|
||||
additional_drop_params=kwargs.get("additional_drop_params"),
|
||||
additional_drop_params=additional_drop_params,
|
||||
)
|
||||
|
||||
if image_edit_provider_config.use_multipart_form_data() and (
|
||||
|
|
@ -1039,7 +1064,19 @@ def image_edit(
|
|||
image=images,
|
||||
prompt=prompt,
|
||||
image_edit_provider_config=image_edit_provider_config,
|
||||
image_edit_optional_request_params=image_edit_request_params,
|
||||
image_edit_optional_request_params={
|
||||
**image_edit_request_params,
|
||||
**(
|
||||
_tokenhub_passthrough_edit_params(
|
||||
non_default_params,
|
||||
extra_body if isinstance(extra_body, dict) else None,
|
||||
provider_supported_params,
|
||||
additional_drop_params,
|
||||
)
|
||||
if custom_llm_provider == litellm.LlmProviders.TENCENT.value
|
||||
else {}
|
||||
),
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import base64
|
||||
import os
|
||||
from collections.abc import Collection, Mapping
|
||||
from io import BufferedReader, BytesIO
|
||||
from typing import Any, Final, cast, get_type_hints
|
||||
from pathlib import Path
|
||||
from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
|
@ -16,7 +19,7 @@ class ImageEditRequestUtils:
|
|||
image_edit_provider_config: BaseImageEditConfig,
|
||||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
drop_params: bool | None = None,
|
||||
additional_drop_params: list[str] | None = None,
|
||||
additional_drop_params: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Get optional parameters for the image edit API.
|
||||
|
|
@ -145,3 +148,36 @@ class ImageEditRequestUtils:
|
|||
except Exception:
|
||||
# If anything goes wrong, default to PNG
|
||||
return FILE_MIME_TYPES[FileType.PNG]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SeekableBinaryReader(Protocol):
|
||||
def tell(self) -> int: ...
|
||||
|
||||
def seek(self, offset: int) -> int: ...
|
||||
|
||||
def read(self) -> bytes: ...
|
||||
|
||||
|
||||
def _read_image_bytes(image: object) -> bytes:
|
||||
if isinstance(image, bytes):
|
||||
return image
|
||||
if isinstance(image, tuple):
|
||||
return _read_image_bytes(image[1])
|
||||
if isinstance(image, os.PathLike):
|
||||
return Path(image).read_bytes()
|
||||
if isinstance(image, _SeekableBinaryReader):
|
||||
position: Final = image.tell()
|
||||
image.seek(0)
|
||||
data: Final = image.read()
|
||||
image.seek(position)
|
||||
return data
|
||||
raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}")
|
||||
|
||||
|
||||
def image_to_data_url(image: object) -> str:
|
||||
if isinstance(image, str):
|
||||
return image
|
||||
image_bytes: Final = _read_image_bytes(image)
|
||||
mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
|
||||
return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
|
||||
|
|
|
|||
|
|
@ -436,6 +436,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
|
|||
"reset_budget_windows": PostgresOperation("select", None),
|
||||
"write_budget_windows": PostgresOperation("update", None),
|
||||
"roll_window_spend_row": PostgresOperation("update", None),
|
||||
"open_window_spend_row": PostgresOperation("upsert", None),
|
||||
"seed_window_spend": PostgresOperation("select", None),
|
||||
"select_window_spend_rows": PostgresOperation("select", None),
|
||||
"commit_window_spend_updates": PostgresOperation("upsert", None),
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
|
|||
|
||||
_RESPONSE_CONTENT_FIELD: Final = "content"
|
||||
|
||||
_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
|
||||
|
||||
_ResponseT: Final = TypeVar("_ResponseT")
|
||||
|
||||
|
||||
|
|
@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
get_last_user_message,
|
||||
)
|
||||
|
||||
query: Final = get_last_user_message(cast(list[AllMessageValues], messages))
|
||||
last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or ""
|
||||
query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip()
|
||||
if not query:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.interactions import (
|
||||
InteractionsAPIStreamingResponse,
|
||||
)
|
||||
|
|
@ -145,7 +146,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
|||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
self.stream_iterator = response.aiter_lines()
|
||||
self.stream_iterator = aiter_sse_lines(response.aiter_text())
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
|
@ -212,7 +213,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
|
|||
litellm_metadata=litellm_metadata,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
self.stream_iterator = response.iter_lines()
|
||||
self.stream_iterator = iter_sse_lines(response.iter_text())
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"rpm",
|
||||
"itpm",
|
||||
"otpm",
|
||||
"tpd",
|
||||
"rpd",
|
||||
"use_xai_oauth",
|
||||
"chatgpt_auth_file",
|
||||
"github_copilot_auth_type",
|
||||
|
|
|
|||
|
|
@ -437,7 +437,6 @@ def get_llm_provider(
|
|||
model in litellm.open_ai_chat_completion_models
|
||||
or "ft:gpt-3.5-turbo" in model
|
||||
or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o
|
||||
or model in litellm.openai_image_generation_models
|
||||
or model.startswith("gpt-image")
|
||||
or model in litellm.openai_video_generation_models
|
||||
):
|
||||
|
|
|
|||
|
|
@ -2758,7 +2758,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if entry is None or not is_str_keyed_dict(litellm_params):
|
||||
return
|
||||
metadata: Final[object] = litellm_params.get("metadata")
|
||||
bucket: Final = metadata if is_str_keyed_dict(metadata) else {}
|
||||
bucket: Final = {**metadata} if is_str_keyed_dict(metadata) else {}
|
||||
litellm_params["metadata"] = bucket
|
||||
with_guardrail_entry(bucket, entry)
|
||||
|
||||
|
|
@ -3481,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
print_verbose=print_verbose,
|
||||
level=LogfireLevel.INFO.value,
|
||||
level=LogfireLevel.INFO,
|
||||
)
|
||||
|
||||
if callback == "lunary" and lunaryLogger is not None:
|
||||
|
|
@ -4442,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
level=LogfireLevel.ERROR.value,
|
||||
level=LogfireLevel.ERROR,
|
||||
print_verbose=print_verbose,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timezone, tzinfo
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypedDict, cast
|
||||
from typing import Final, Literal, Protocol, TypedDict, cast
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from typing_extensions import ReadOnly
|
||||
|
|
@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost(
|
|||
)
|
||||
|
||||
|
||||
class _ModelPricedImageCostCalculator(Protocol):
|
||||
def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ...
|
||||
|
||||
|
||||
def _model_priced_image_cost_calculator(
|
||||
custom_llm_provider: str | None, model: str
|
||||
) -> _ModelPricedImageCostCalculator | None:
|
||||
match custom_llm_provider:
|
||||
case litellm.LlmProviders.RECRAFT.value:
|
||||
from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator
|
||||
|
||||
return recraft_image_cost_calculator
|
||||
case litellm.LlmProviders.AIML.value:
|
||||
from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator
|
||||
|
||||
return aiml_image_cost_calculator
|
||||
case litellm.LlmProviders.COMETAPI.value:
|
||||
from litellm.llms.cometapi.image_generation.cost_calculator import (
|
||||
cost_calculator as cometapi_image_cost_calculator,
|
||||
)
|
||||
|
||||
return cometapi_image_cost_calculator
|
||||
case litellm.LlmProviders.RUNWAYML.value:
|
||||
from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator
|
||||
|
||||
return runwayml_image_cost_calculator
|
||||
case litellm.LlmProviders.TENCENT.value:
|
||||
from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator
|
||||
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
|
||||
|
||||
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
class CostCalculatorUtils:
|
||||
@staticmethod
|
||||
def call_type_has_image_response(call_type: str) -> bool:
|
||||
|
|
@ -2131,9 +2166,6 @@ class CostCalculatorUtils:
|
|||
from litellm.llms.gemini.image_generation.cost_calculator import (
|
||||
cost_calculator as gemini_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
|
||||
cost_calculator as vertex_ai_image_cost_calculator,
|
||||
)
|
||||
|
|
@ -2146,7 +2178,10 @@ class CostCalculatorUtils:
|
|||
)
|
||||
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
|
||||
pricing: Final = deployment_pricing(model_info)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
|
||||
|
||||
if model_priced_calculator is not None:
|
||||
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
|
||||
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
|
||||
if isinstance(completion_response, ImageResponse):
|
||||
return vertex_ai_image_cost_calculator(
|
||||
|
|
@ -2164,36 +2199,6 @@ class CostCalculatorUtils:
|
|||
optional_params=optional_params,
|
||||
)
|
||||
raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation")
|
||||
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
|
||||
return recraft_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.AIML.value:
|
||||
from litellm.llms.aiml.image_generation.cost_calculator import (
|
||||
cost_calculator as aiml_image_cost_calculator,
|
||||
)
|
||||
|
||||
return aiml_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
|
||||
from litellm.llms.cometapi.image_generation.cost_calculator import (
|
||||
cost_calculator as cometapi_image_cost_calculator,
|
||||
)
|
||||
|
||||
return cometapi_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
|
||||
if call_type in (
|
||||
CallTypes.image_edit.value,
|
||||
|
|
@ -2237,16 +2242,6 @@ class CostCalculatorUtils:
|
|||
optional_params=optional_params,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
|
||||
from litellm.llms.runwayml.cost_calculator import (
|
||||
cost_calculator as runwayml_image_cost_calculator,
|
||||
)
|
||||
|
||||
return runwayml_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
model_info=pricing,
|
||||
)
|
||||
elif (
|
||||
custom_llm_provider == litellm.LlmProviders.OPENAI.value
|
||||
or custom_llm_provider == litellm.LlmProviders.AZURE.value
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.batch_utils import InMemoryFile
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.router_utils.batch_utils import InMemoryFile
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ def safe_json_structure(
|
|||
def _transform(key: str | None, value: str) -> str:
|
||||
return value if value_transform is None else value_transform(key, value)
|
||||
|
||||
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
|
||||
def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
|
||||
# Check for maximum depth.
|
||||
if depth > max_depth:
|
||||
return "MaxDepthExceeded"
|
||||
|
|
@ -48,32 +48,32 @@ def safe_json_structure(
|
|||
if id(obj) in seen:
|
||||
return "CircularReference Detected"
|
||||
seen.add(id(obj))
|
||||
result: dict | list | tuple | set | str
|
||||
if isinstance(obj, dict):
|
||||
result = {}
|
||||
for k, v in obj.items():
|
||||
if isinstance(k, (str)):
|
||||
clean_k = k.replace("\x00", "") if "\x00" in k else k
|
||||
result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
|
||||
mapping_result: Final = {
|
||||
(clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
|
||||
for k, v in obj.items()
|
||||
if isinstance(k, str)
|
||||
}
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return mapping_result
|
||||
elif isinstance(obj, list):
|
||||
result = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return list_result
|
||||
elif isinstance(obj, tuple):
|
||||
result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
|
||||
tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return tuple_result
|
||||
elif isinstance(obj, set):
|
||||
result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
|
||||
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
set_result: Final = sorted(sortable_values)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return set_result
|
||||
elif isinstance(obj, BaseModel):
|
||||
dumped: Final = obj.model_dump()
|
||||
result = _serialize(dumped, seen, depth + 1, key)
|
||||
model_result: Final = _serialize(dumped, seen, depth + 1, key)
|
||||
seen.remove(id(obj))
|
||||
return result
|
||||
return model_result
|
||||
else:
|
||||
# Fall back to string conversion for non-serializable objects.
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1263,7 +1263,7 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
if _is_delta_empty:
|
||||
model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned
|
||||
model_response.choices[0].delta = Delta(content=self._close_open_think_block())
|
||||
# get any function call arguments
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
finish_reason=self.received_finish_reason
|
||||
|
|
@ -1292,21 +1292,26 @@ class CustomStreamWrapper:
|
|||
if self.merge_reasoning_content_in_choices is not True:
|
||||
return
|
||||
delta: Final = model_response.choices[0].delta
|
||||
reasoning_content: Final = getattr(delta, "reasoning_content", None)
|
||||
reasoning_content: Final = getattr(delta, "reasoning_content", None) or ""
|
||||
content: Final = delta.content or ""
|
||||
if reasoning_content:
|
||||
self._emitted_disqualifying_content = True
|
||||
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
|
||||
opening: Final = "" if self.sent_first_thinking_block else "<think>"
|
||||
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
|
||||
self.sent_first_thinking_block = True
|
||||
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
|
||||
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
|
||||
delta.content = "</think>" + content
|
||||
self.sent_last_thinking_block = True
|
||||
|
||||
ends_thinking: Final = bool(content or delta.tool_calls or delta.function_call)
|
||||
inside_think_block: Final = self.sent_first_thinking_block and not self.sent_last_thinking_block
|
||||
opening: Final = "<think>" if reasoning_content and not inside_think_block else ""
|
||||
closing: Final = "</think>" if ends_thinking and (reasoning_content or inside_think_block) else ""
|
||||
if hasattr(delta, "reasoning_content"):
|
||||
del delta.reasoning_content
|
||||
if not reasoning_content and not closing:
|
||||
return
|
||||
delta.content = opening + reasoning_content + closing + content
|
||||
self.sent_first_thinking_block = True
|
||||
self.sent_last_thinking_block = bool(closing)
|
||||
self._emitted_disqualifying_content = self._emitted_disqualifying_content or bool(reasoning_content)
|
||||
|
||||
def _close_open_think_block(self) -> str | None:
|
||||
if not self.sent_first_thinking_block or self.sent_last_thinking_block:
|
||||
return None
|
||||
self.sent_last_thinking_block = True
|
||||
return "</think>"
|
||||
|
||||
def _dispatch_provider_chunk(
|
||||
self,
|
||||
|
|
@ -1932,8 +1937,20 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].finish_reason == "stop" and self.tool_call
|
||||
): # don't overwrite for other - potential error finish reasons
|
||||
model_response.choices[0].finish_reason = "tool_calls"
|
||||
close_tag: Final = self._close_open_think_block()
|
||||
if close_tag is not None:
|
||||
model_response.choices[0].delta.content = close_tag
|
||||
return model_response
|
||||
|
||||
def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
return processed_chunk
|
||||
if processed_chunk.choices[0].delta.content is None:
|
||||
return None
|
||||
return processed_chunk.model_copy(
|
||||
update={"choices": [processed_chunk.choices[0].model_copy(update={"finish_reason": None})]}
|
||||
)
|
||||
|
||||
def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
|
||||
"""
|
||||
Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
|
||||
|
|
@ -2173,10 +2190,9 @@ class CustomStreamWrapper:
|
|||
else:
|
||||
self.sent_last_chunk = True
|
||||
processed_chunk: Final = self.finish_reason_handler()
|
||||
# The logged response is built from self.chunks; keep a finish_reason the provider sent on its
|
||||
# last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
self.chunks.append(processed_chunk)
|
||||
final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
|
||||
if final_chunk_for_logging is not None:
|
||||
self.chunks.append(final_chunk_for_logging)
|
||||
if self.stream_options is None:
|
||||
self._set_final_chunk_usage(processed_chunk)
|
||||
## LOGGING
|
||||
|
|
@ -2406,10 +2422,9 @@ class CustomStreamWrapper:
|
|||
else:
|
||||
self.sent_last_chunk = True
|
||||
processed_chunk: Final = self.finish_reason_handler()
|
||||
# The logged response is built from self.chunks; keep a finish_reason the provider sent on its
|
||||
# last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
self.chunks.append(processed_chunk)
|
||||
final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
|
||||
if final_chunk_for_logging is not None:
|
||||
self.chunks.append(final_chunk_for_logging)
|
||||
if self.stream_options is None:
|
||||
self._set_final_chunk_usage(processed_chunk)
|
||||
# see sync __next__'s sibling branch: deliberately do NOT restore
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
get_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.llms.anthropic import (
|
||||
ContentBlockDelta,
|
||||
ContentBlockStart,
|
||||
|
|
@ -233,7 +234,7 @@ async def make_call(
|
|||
raise AnthropicError(status_code=500, message=str(e))
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
json_mode=json_mode,
|
||||
speed=speed,
|
||||
|
|
@ -304,7 +305,7 @@ def make_sync_call(
|
|||
)
|
||||
|
||||
completion_stream: Final = ModelResponseIterator(
|
||||
streaming_response=response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(response.iter_text()),
|
||||
sync_stream=True,
|
||||
json_mode=json_mode,
|
||||
speed=speed,
|
||||
|
|
|
|||
|
|
@ -268,6 +268,7 @@ async def caller_within_model_budget(
|
|||
user_id=user_id,
|
||||
user_model_max_budget=user_model_max_budget,
|
||||
model=model,
|
||||
team_id=user_api_key_auth.team_id,
|
||||
)
|
||||
except litellm.BudgetExceededError:
|
||||
return "denied"
|
||||
|
|
@ -309,6 +310,7 @@ async def caller_within_model_budget(
|
|||
end_user_id=end_user_id,
|
||||
end_user_model_max_budget=end_user_model_max_budget,
|
||||
model=model,
|
||||
team_id=user_api_key_auth.team_id,
|
||||
)
|
||||
except litellm.BudgetExceededError:
|
||||
return "denied"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.llms.anthropic.chat.handler import (
|
|||
)
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
Message,
|
||||
|
|
@ -269,7 +270,7 @@ def build_complete_streaming_response_legacy(
|
|||
|
||||
def extract_sse_data(event_str: str) -> dict | None:
|
||||
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
|
||||
for line in event_str.splitlines():
|
||||
for line in split_sse_lines(event_str):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("data:"):
|
||||
payload = stripped[len("data:") :].strip()
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
|
|||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
|
||||
from litellm.router_utils.baseline_request import (
|
||||
from litellm.router_strategy.auto_router.baseline_request import (
|
||||
BASELINE_PARAMETERS,
|
||||
capture_baseline_parameters,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,13 +13,12 @@ from openai import (
|
|||
BadRequestError,
|
||||
OpenAI,
|
||||
)
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
|
||||
from litellm.constants import DEFAULT_MAX_RETRIES
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -34,7 +33,6 @@ from litellm.types.utils import (
|
|||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
convert_to_model_response_object,
|
||||
modify_url,
|
||||
)
|
||||
|
||||
from ...types.llms.openai import HttpxBinaryResponseContent
|
||||
|
|
@ -59,8 +57,6 @@ from .image_generation import (
|
|||
)
|
||||
from .image_generation.http_utils import azure_deployment_image_generation_json_body
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AzureOpenAIAssistantsAPIConfig:
|
||||
"""
|
||||
|
|
@ -860,11 +856,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
headers: dict,
|
||||
deployment_name: str | None = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Implemented for azure dall-e-2 image gen calls
|
||||
|
||||
Alternative to needing a custom transport implementation
|
||||
"""
|
||||
if client is None:
|
||||
_params: Final = {}
|
||||
if timeout is not None:
|
||||
|
|
@ -881,95 +872,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
else:
|
||||
async_handler = client
|
||||
|
||||
if (
|
||||
"images/generations" in api_base
|
||||
and api_version
|
||||
in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
|
||||
"2023-06-01-preview",
|
||||
"2023-07-01-preview",
|
||||
"2023-08-01-preview",
|
||||
"2023-09-01-preview",
|
||||
"2023-10-01-preview",
|
||||
]
|
||||
): # CREATE + POLL for azure dall-e-2 calls
|
||||
api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
|
||||
|
||||
data.pop(
|
||||
"model", None
|
||||
) # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
|
||||
response = await async_handler.post(
|
||||
url=api_base,
|
||||
data=json.dumps(data),
|
||||
headers=headers,
|
||||
)
|
||||
if "operation-location" in response.headers:
|
||||
operation_location_url: Final = response.headers["operation-location"]
|
||||
else:
|
||||
raise AzureOpenAIError(status_code=500, message=response.text)
|
||||
# Reject polling URLs that don't share an origin with ``api_base``.
|
||||
# Without this an upstream-controlled or attacker-controlled
|
||||
# value would receive the operator's Azure API key in the
|
||||
# request headers below. VERIA-51.
|
||||
try:
|
||||
assert_same_origin(operation_location_url, api_base)
|
||||
except SSRFError as ssrf_err:
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message=f"Rejected polling URL: {ssrf_err}",
|
||||
)
|
||||
response = await async_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
await response.aread()
|
||||
|
||||
timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
|
||||
start_time: Final = time.time()
|
||||
if "status" not in response.json():
|
||||
# Don't reflect the raw response body — when the polling
|
||||
# URL points at an internal JSON API (cloud metadata
|
||||
# service etc.) reflecting it here turns Blind SSRF into
|
||||
# Full-Read SSRF. VERIA-51.
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message="Polling response missing 'status' field",
|
||||
)
|
||||
while response.json()["status"] not in ["succeeded", "failed"]:
|
||||
if time.time() - start_time > timeout_secs:
|
||||
raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
|
||||
|
||||
await asyncio.sleep(int(response.headers.get("retry-after") or 10))
|
||||
response = await async_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
await response.aread()
|
||||
|
||||
if response.json()["status"] == "failed":
|
||||
error_data: Final = response.json()
|
||||
# Preserve Azure error details (e.g. content_policy_violation,
|
||||
# inner_error, content_filter_results) as structured body so
|
||||
# exception_type() can route them correctly.
|
||||
_error_body: Final = error_data.get("error", error_data)
|
||||
_error_msg: Final = (
|
||||
_error_body.get("message", "Image generation failed")
|
||||
if isinstance(_error_body, dict)
|
||||
else json.dumps(error_data)
|
||||
)
|
||||
raise AzureOpenAIError(
|
||||
status_code=400,
|
||||
message=_error_msg,
|
||||
body=error_data,
|
||||
)
|
||||
|
||||
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
headers=response.headers,
|
||||
content=json.dumps(result).encode("utf-8"),
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
|
||||
)
|
||||
request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
|
||||
return await async_handler.post(
|
||||
url=api_base,
|
||||
|
|
@ -988,11 +890,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
headers: dict,
|
||||
deployment_name: str | None = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Implemented for azure dall-e-2 image gen calls
|
||||
|
||||
Alternative to needing a custom transport implementation
|
||||
"""
|
||||
if client is None:
|
||||
_params: Final = {}
|
||||
if timeout is not None:
|
||||
|
|
@ -1006,87 +903,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
else:
|
||||
sync_handler = client
|
||||
|
||||
if (
|
||||
"images/generations" in api_base
|
||||
and api_version
|
||||
in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
|
||||
"2023-06-01-preview",
|
||||
"2023-07-01-preview",
|
||||
"2023-08-01-preview",
|
||||
"2023-09-01-preview",
|
||||
"2023-10-01-preview",
|
||||
]
|
||||
): # CREATE + POLL for azure dall-e-2 calls
|
||||
api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
|
||||
|
||||
data.pop(
|
||||
"model", None
|
||||
) # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
|
||||
response = sync_handler.post(
|
||||
url=api_base,
|
||||
data=json.dumps(data),
|
||||
headers=headers,
|
||||
)
|
||||
if "operation-location" in response.headers:
|
||||
operation_location_url: Final = response.headers["operation-location"]
|
||||
else:
|
||||
raise AzureOpenAIError(status_code=500, message=response.text)
|
||||
try:
|
||||
assert_same_origin(operation_location_url, api_base)
|
||||
except SSRFError as ssrf_err:
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message=f"Rejected polling URL: {ssrf_err}",
|
||||
)
|
||||
response = sync_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
response.read()
|
||||
|
||||
timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
|
||||
start_time: Final = time.time()
|
||||
if "status" not in response.json():
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message="Polling response missing 'status' field",
|
||||
)
|
||||
while response.json()["status"] not in ["succeeded", "failed"]:
|
||||
if time.time() - start_time > timeout_secs:
|
||||
raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
|
||||
|
||||
time.sleep(int(response.headers.get("retry-after") or 10))
|
||||
response = sync_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
response.read()
|
||||
|
||||
if response.json()["status"] == "failed":
|
||||
error_data: Final = response.json()
|
||||
# Preserve Azure error details (e.g. content_policy_violation,
|
||||
# inner_error, content_filter_results) as structured body so
|
||||
# exception_type() can route them correctly.
|
||||
_error_body: Final = error_data.get("error", error_data)
|
||||
_error_msg: Final = (
|
||||
_error_body.get("message", "Image generation failed")
|
||||
if isinstance(_error_body, dict)
|
||||
else json.dumps(error_data)
|
||||
)
|
||||
raise AzureOpenAIError(
|
||||
status_code=400,
|
||||
message=_error_msg,
|
||||
body=error_data,
|
||||
)
|
||||
|
||||
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
headers=response.headers,
|
||||
content=json.dumps(result).encode("utf-8"),
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
|
||||
)
|
||||
request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
|
||||
return sync_handler.post(
|
||||
url=api_base,
|
||||
|
|
@ -1193,10 +1009,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
deployment_name=model,
|
||||
)
|
||||
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
|
||||
if provider_config is not None:
|
||||
return provider_config.transform_image_generation_response(
|
||||
model=data.get("model", "dall-e-2"),
|
||||
model=data.get("model", ""),
|
||||
raw_response=httpx_response,
|
||||
model_response=model_response or ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
|
|
@ -1329,10 +1145,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
headers=request_headers,
|
||||
deployment_name=model,
|
||||
)
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
|
||||
if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):
|
||||
return provider_config.transform_image_generation_response(
|
||||
model=data.get("model", "dall-e-2"),
|
||||
model=data.get("model", ""),
|
||||
raw_response=httpx_response,
|
||||
model_response=model_response or ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -4,14 +4,10 @@ from litellm.llms.base_llm.image_generation.transformation import (
|
|||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import AzureDallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import AzureDallE3ImageGenerationConfig
|
||||
from .gpt_transformation import AzureGPTImageGenerationConfig
|
||||
from .http_utils import azure_deployment_image_generation_json_body
|
||||
|
||||
__all__ = [
|
||||
"AzureDallE2ImageGenerationConfig",
|
||||
"AzureDallE3ImageGenerationConfig",
|
||||
"AzureGPTImageGenerationConfig",
|
||||
"azure_deployment_image_generation_json_body",
|
||||
]
|
||||
|
|
@ -21,11 +17,7 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
|||
model = model.lower()
|
||||
model = model.replace("-", "")
|
||||
model = model.replace("_", "")
|
||||
if model == "" or "dalle2" in model: # empty model is dall-e-2
|
||||
return AzureDallE2ImageGenerationConfig()
|
||||
elif "dalle3" in model:
|
||||
return AzureDallE3ImageGenerationConfig()
|
||||
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
return AzureFoundryMAIImageGenerationConfig()
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-2 image generation config
|
||||
"""
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-3 image generation config
|
||||
"""
|
||||
|
|
@ -35,6 +35,7 @@ from litellm.llms.azure_ai.agents.transformation import (
|
|||
AzureAIAgentsConfig,
|
||||
AzureAIAgentsError,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionAnnotationURLCitation,
|
||||
|
|
@ -632,7 +633,7 @@ class AzureAIAgentsHandler:
|
|||
|
||||
current_event = None
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
async for line in aiter_sse_lines(response.aiter_text()):
|
||||
line = line.strip()
|
||||
|
||||
if line.startswith("event:"):
|
||||
|
|
|
|||
|
|
@ -3,15 +3,11 @@ from litellm.llms.base_llm.image_generation.transformation import (
|
|||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import AzureFoundryDallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import AzureFoundryDallE3ImageGenerationConfig
|
||||
from .flux_transformation import AzureFoundryFluxImageGenerationConfig
|
||||
from .gpt_transformation import AzureFoundryGPTImageGenerationConfig
|
||||
from .mai_transformation import AzureFoundryMAIImageGenerationConfig
|
||||
|
||||
__all__ = [
|
||||
"AzureFoundryDallE2ImageGenerationConfig",
|
||||
"AzureFoundryDallE3ImageGenerationConfig",
|
||||
"AzureFoundryFluxImageGenerationConfig",
|
||||
"AzureFoundryGPTImageGenerationConfig",
|
||||
"AzureFoundryMAIImageGenerationConfig",
|
||||
|
|
@ -22,11 +18,7 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi
|
|||
model = model.lower()
|
||||
model = model.replace("-", "")
|
||||
model = model.replace("_", "")
|
||||
if model == "" or "dalle2" in model: # empty model is dall-e-2
|
||||
return AzureFoundryDallE2ImageGenerationConfig()
|
||||
elif "dalle3" in model:
|
||||
return AzureFoundryDallE3ImageGenerationConfig()
|
||||
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
return AzureFoundryMAIImageGenerationConfig()
|
||||
elif "flux" in model:
|
||||
return AzureFoundryFluxImageGenerationConfig()
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-2 image generation config
|
||||
"""
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-3 image generation config
|
||||
"""
|
||||
|
|
@ -96,7 +96,7 @@ class BaseModelResponseIterator:
|
|||
released (and a backend like vLLM aborts generation) when the stream
|
||||
is abandoned before its natural end.
|
||||
|
||||
``streaming_response`` is usually a bare ``aiter_lines()`` generator
|
||||
``streaming_response`` is usually a bare ``aiter_sse_lines()`` generator
|
||||
that holds no reference to the response, so the handler that owns the
|
||||
response attaches it here after construction."""
|
||||
if self.http_response is not None:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row ha
|
|||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
|
||||
|
||||
|
|
@ -24,7 +25,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]:
|
|||
|
||||
events: Final[list[dict]] = []
|
||||
for block in chunk.split("\n\n"):
|
||||
for line in block.splitlines():
|
||||
for line in split_sse_lines(block):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("data:"):
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Final
|
|||
import httpx
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
SANDBOX_MAX_OUTPUT_BYTES: Final = 10 * 1024 * 1024
|
||||
|
|
@ -99,7 +100,7 @@ class BaseSandboxConfig:
|
|||
async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
|
||||
lines: Final[list[str]] = []
|
||||
total = 0
|
||||
async for line in response.aiter_lines():
|
||||
async for line in aiter_sse_lines(response.aiter_text()):
|
||||
total += len(line.encode("utf-8"))
|
||||
if total > SANDBOX_MAX_OUTPUT_BYTES:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import time
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -34,6 +36,15 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _PegasusResponse(TypedDict):
|
||||
message: ReadOnly[NotRequired[str | None]]
|
||||
finishReason: ReadOnly[NotRequired[str | None]]
|
||||
|
||||
|
||||
_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
|
||||
|
||||
|
||||
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
||||
"""
|
||||
Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
|
||||
|
|
@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
ModelResponse with choices[0].message.content and finish_reason
|
||||
"""
|
||||
try:
|
||||
completion_response: Final = raw_response.json()
|
||||
completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
|
||||
except Exception as e:
|
||||
raise BedrockError(
|
||||
message=f"Error parsing response: {raw_response.text}, error: {e}",
|
||||
|
|
@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
message_content: Final = completion_response.get("message", "")
|
||||
|
||||
# Extract finish reason and map to LiteLLM format
|
||||
finish_reason_raw: Final = completion_response.get("finishReason", "stop")
|
||||
finish_reason_raw: Final = completion_response.get("finishReason") or "stop"
|
||||
finish_reason: Final = map_finish_reason(finish_reason_raw)
|
||||
|
||||
# Set the response content
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import copy
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
|||
|
||||
_GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _CohereGeneration(TypedDict):
|
||||
text: ReadOnly[str | None]
|
||||
finish_reason: ReadOnly[str | None]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _CohereInvokeResponse(TypedDict):
|
||||
text: NotRequired[ReadOnly[str | None]]
|
||||
generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _LlamaInvokeResponse(TypedDict):
|
||||
generation: ReadOnly[str | None]
|
||||
|
||||
|
||||
_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
|
||||
_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
|
||||
|
||||
_GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
||||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
|
@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
outputText: str | None = None
|
||||
try:
|
||||
if provider == "cohere":
|
||||
if "text" in completion_response:
|
||||
outputText = completion_response["text"]
|
||||
elif "generations" in completion_response:
|
||||
outputText = completion_response["generations"][0]["text"]
|
||||
cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
|
||||
if "text" in cohere_response:
|
||||
outputText = cohere_response["text"]
|
||||
elif "generations" in cohere_response:
|
||||
outputText = cohere_response["generations"][0]["text"]
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
completion_response["generations"][0]["finish_reason"]
|
||||
cohere_response["generations"][0]["finish_reason"] or "stop"
|
||||
)
|
||||
elif provider == "anthropic":
|
||||
return litellm.AmazonAnthropicClaudeConfig().transform_response(
|
||||
|
|
@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
elif provider == "ai21":
|
||||
outputText = completion_response.get("completions")[0].get("data").get("text")
|
||||
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
|
||||
outputText = completion_response["generation"]
|
||||
outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
|
||||
elif provider == "mistral":
|
||||
outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
|
||||
else: # amazon titan
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.llms.base_llm.files.transformation import (
|
|||
BaseFilesConfig,
|
||||
LiteLLMLoggingObj,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.bedrock import AwsAuthParams, BedrockBatchRecordKind
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -400,7 +401,7 @@ def _batch_input_line(line_number: int, line: str) -> _BatchInputLine:
|
|||
|
||||
|
||||
def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]:
|
||||
numbered_lines: Final = enumerate(file_content.splitlines(), start=1)
|
||||
numbered_lines: Final = enumerate(split_sse_lines(file_content), start=1)
|
||||
return ((line_number, line) for line_number, line in numbered_lines if line.strip())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.llms.bedrock.passthrough.guardrail_translation.structured_edits imp
|
|||
structured_edits,
|
||||
tool_edits,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy.guardrails.auto_router_compression import is_compression_guardrail
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
|
|
@ -947,7 +948,7 @@ def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]:
|
|||
def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
_JSON_OBJECT.validate_python(json.loads(line[len("data:") :]))
|
||||
for line in sse.decode().splitlines()
|
||||
for line in split_sse_lines(sse.decode())
|
||||
if line.startswith("data:")
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.types.realtime import (
|
|||
RealtimeResponseTypedDict,
|
||||
)
|
||||
|
||||
_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
|
@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
List of Bedrock format messages (JSON strings)
|
||||
"""
|
||||
try:
|
||||
json_message: Final[dict[str, JsonValue]] = json.loads(message)
|
||||
json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
|
||||
except json.JSONDecodeError:
|
||||
verbose_logger.warning("Invalid JSON message: %s", message[:200])
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.llms.base_llm.search.transformation import (
|
|||
)
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
# AgentCore web-search rejects queries longer than 200 characters
|
||||
|
|
@ -146,7 +147,9 @@ def _iter_sse_events(text: str) -> Iterator[Mapping[str, object]]:
|
|||
progress notifications before the JSON-RPC response.
|
||||
"""
|
||||
for chunk in _SSE_EVENT_SEPARATOR.split(text):
|
||||
payload = "\n".join(line[len("data:") :].lstrip() for line in chunk.splitlines() if line.startswith("data:"))
|
||||
payload = "\n".join(
|
||||
line[len("data:") :].lstrip() for line in split_sse_lines(chunk) if line.startswith("data:")
|
||||
)
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
safe_convert_created_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.llms.openai.common_utils import OpenAIError
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.responses.sse_output_recovery import (
|
||||
|
|
@ -199,7 +200,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
error_message = None
|
||||
streamed_output_items: Final[dict[int, dict[str, object]]] = {}
|
||||
text_only_output_items: Final[dict[int, dict[str, object]]] = {}
|
||||
for chunk in body_text.splitlines():
|
||||
for chunk in split_sse_lines(body_text):
|
||||
parsed_chunk = parse_sse_json_chunk(chunk)
|
||||
if parsed_chunk is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.utils import TextChoices
|
||||
from litellm.utils import CustomStreamWrapper, TextCompletionResponse
|
||||
|
||||
|
|
@ -109,7 +110,7 @@ async def make_call(
|
|||
if response.status_code != 200:
|
||||
raise TextCompletionCodestralError(status_code=response.status_code, message=response.text)
|
||||
|
||||
completion_stream: Final = response.aiter_lines()
|
||||
completion_stream: Final = aiter_sse_lines(response.aiter_text())
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
|
|
@ -339,13 +340,13 @@ class CodestralTextCompletion:
|
|||
response = litellm.module_level_client.post(
|
||||
completion_url,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
data=json.dumps({**data, "stream": True}),
|
||||
stream=stream,
|
||||
)
|
||||
_response: Final = CustomStreamWrapper(
|
||||
response.iter_lines(),
|
||||
iter_sse_lines(response.iter_text()),
|
||||
model,
|
||||
custom_llm_provider="codestral",
|
||||
custom_llm_provider="text-completion-codestral",
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return _response
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_httpx_client,
|
||||
get_ssl_configuration,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
|
||||
from litellm.types.llms.openai import FileTypes
|
||||
from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
|
||||
|
|
@ -466,7 +467,7 @@ class BaseLLMAIOHTTPHandler:
|
|||
)
|
||||
else:
|
||||
completion_stream = provider_config.get_model_response_iterator(
|
||||
streaming_response=response.iter_lines(), sync_stream=True
|
||||
streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
|
|
@ -560,11 +561,8 @@ class BaseLLMAIOHTTPHandler:
|
|||
organization: str | None = None,
|
||||
headers: dict | None = None,
|
||||
) -> ImageResponse:
|
||||
if model is None:
|
||||
raise ValueError("model is required for non-openai image variations")
|
||||
|
||||
provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
|
||||
model=model, # openai defaults to dall-e-2
|
||||
model=model or "",
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import ssl
|
|||
import sys
|
||||
import typing
|
||||
import urllib.request
|
||||
import weakref
|
||||
from collections.abc import Callable, Generator
|
||||
from typing import ClassVar, Final
|
||||
|
||||
|
|
@ -109,6 +110,24 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
|
|||
raise mapped_exc(message) from exc
|
||||
|
||||
|
||||
def _release_transport_anchor(_transport: object) -> None:
|
||||
"""Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
|
||||
|
||||
|
||||
def _anchor_transport_to(stream: "AiohttpResponseStream", transport: object) -> None:
|
||||
"""Keep the transport alive for as long as the stream can still read.
|
||||
|
||||
The transport finalizer closes the session it owns, and nothing on the path from a
|
||||
live response back to that session references the transport, so a client dropped
|
||||
mid-stream would close the session under the body. ``weakref.finalize`` holds the
|
||||
transport in its own registry rather than on the stream, which keeps the transport
|
||||
out of the response's reference cycle (an exception carrying the response, for one)
|
||||
so it is finalized by refcount once the anchor drops and can still schedule an async
|
||||
close, instead of being collected in the same cyclic pass as the session it closes.
|
||||
"""
|
||||
weakref.finalize(stream, _release_transport_anchor, transport)
|
||||
|
||||
|
||||
class AiohttpResponseStream(httpx.AsyncByteStream):
|
||||
CHUNK_SIZE = 1024 * 16
|
||||
|
||||
|
|
@ -463,10 +482,12 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
# Re-raise if it's a different RuntimeError
|
||||
raise
|
||||
|
||||
stream: Final = AiohttpResponseStream(response)
|
||||
_anchor_transport_to(stream, self)
|
||||
return httpx.Response(
|
||||
status_code=response.status,
|
||||
headers=response.headers,
|
||||
stream=AiohttpResponseStream(response),
|
||||
stream=stream,
|
||||
request=request,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_httpx_client,
|
||||
header_value,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.responses.streaming_iterator import (
|
||||
BaseResponsesAPIStreamingIterator,
|
||||
MockResponsesAPIStreamingIterator,
|
||||
|
|
@ -1086,7 +1087,7 @@ class BaseLLMHTTPHandler:
|
|||
completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
|
||||
else:
|
||||
completion_stream = provider_config.get_model_response_iterator_for_request(
|
||||
streaming_response=response.iter_lines(),
|
||||
streaming_response=iter_sse_lines(response.iter_text()),
|
||||
sync_stream=True,
|
||||
json_mode=json_mode,
|
||||
optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),
|
||||
|
|
@ -1229,7 +1230,7 @@ class BaseLLMHTTPHandler:
|
|||
completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
|
||||
else:
|
||||
completion_stream = provider_config.get_model_response_iterator_for_request(
|
||||
streaming_response=response.aiter_lines(),
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()),
|
||||
sync_stream=False,
|
||||
json_mode=json_mode,
|
||||
optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),
|
||||
|
|
|
|||
74
litellm/llms/custom_httpx/sse_lines.py
Normal file
74
litellm/llms/custom_httpx/sse_lines.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
Split a decoded Server-Sent Events body into lines the way the SSE spec does.
|
||||
|
||||
httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028,
|
||||
U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a
|
||||
`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR
|
||||
(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream)
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
|
||||
from typing import Final
|
||||
|
||||
_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
|
||||
_SSE_LINE_WITH_BREAK: Final = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+")
|
||||
|
||||
|
||||
def _split_lines(text: str) -> Sequence[str]:
|
||||
if "\r" not in text:
|
||||
return text.split("\n")
|
||||
crlf_folded: Final = text.replace("\r\n", "\n")
|
||||
if "\r" not in crlf_folded:
|
||||
return crlf_folded.split("\n")
|
||||
return _SSE_LINE_BREAK.split(text)
|
||||
|
||||
|
||||
class _SSELineSplitter:
|
||||
def __init__(self) -> None:
|
||||
self._partial_line: io.StringIO = io.StringIO()
|
||||
self._after_cr: bool = False
|
||||
|
||||
def feed(self, text: str) -> tuple[str, ...]:
|
||||
if not text:
|
||||
return ()
|
||||
fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text
|
||||
self._after_cr = fresh.endswith("\r")
|
||||
*ended, tail = _split_lines(fresh)
|
||||
if not ended:
|
||||
_ = self._partial_line.write(tail)
|
||||
return ()
|
||||
first: Final = self._partial_line.getvalue() + ended[0]
|
||||
self._partial_line = io.StringIO()
|
||||
_ = self._partial_line.write(tail)
|
||||
return (first, *ended[1:])
|
||||
|
||||
def flush(self) -> tuple[str, ...]:
|
||||
rest: Final = self._partial_line.getvalue()
|
||||
self._partial_line = io.StringIO()
|
||||
return (rest,) if rest else ()
|
||||
|
||||
|
||||
def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]:
|
||||
splitter: Final = _SSELineSplitter()
|
||||
for text in text_chunks:
|
||||
yield from splitter.feed(text)
|
||||
yield from splitter.flush()
|
||||
|
||||
|
||||
def split_sse_lines(text: str) -> tuple[str, ...]:
|
||||
return tuple(iter_sse_lines((text,)))
|
||||
|
||||
|
||||
def split_sse_lines_keepends(text: str) -> tuple[str, ...]:
|
||||
return tuple(_SSE_LINE_WITH_BREAK.findall(text))
|
||||
|
||||
|
||||
async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]:
|
||||
splitter: Final = _SSELineSplitter()
|
||||
async for text in text_chunks:
|
||||
for line in splitter.feed(text):
|
||||
yield line
|
||||
for line in splitter.flush():
|
||||
yield line
|
||||
|
|
@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
|
|
@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
|
|||
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _DashScopeUsage(TypedDict, total=False):
|
||||
prompt_tokens: NotRequired[ReadOnly[int]]
|
||||
total_tokens: NotRequired[ReadOnly[int]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _DashScopeEmbeddingResponse(TypedDict, total=False):
|
||||
data: NotRequired[ReadOnly[list[object]]]
|
||||
model: NotRequired[ReadOnly[str | None]]
|
||||
usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
|
||||
id: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
|
||||
|
||||
|
||||
class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
||||
"""
|
||||
Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
|
||||
|
|
@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
|||
message=message,
|
||||
)
|
||||
|
||||
parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
|
||||
model_response.object = "list"
|
||||
model_response.data = response_json.get("data", [])
|
||||
model_response.model = response_json.get("model", model)
|
||||
model_response.data = parsed_response.get("data", [])
|
||||
model_response.model = parsed_response.get("model", model)
|
||||
|
||||
usage: Final = response_json.get("usage") or {}
|
||||
usage: Final = parsed_response.get("usage") or {}
|
||||
prompt_tokens: Final = usage.get("prompt_tokens", 0)
|
||||
total_tokens: Final = usage.get("total_tokens", prompt_tokens)
|
||||
setattr(
|
||||
|
|
@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
|
|||
),
|
||||
)
|
||||
|
||||
if "id" in response_json:
|
||||
setattr(model_response, "id", response_json["id"])
|
||||
if "id" in parsed_response:
|
||||
setattr(model_response, "id", parsed_response["id"])
|
||||
|
||||
return model_response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
|
|||
)
|
||||
|
||||
|
||||
_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
class ModelResponseIterator:
|
||||
def __init__(self, streaming_response, sync_stream: bool):
|
||||
self.streaming_response = streaming_response
|
||||
|
|
@ -99,7 +104,7 @@ class ModelResponseIterator:
|
|||
self.response_iterator = self.streaming_response
|
||||
chunk: Final = self._next_sse_data()
|
||||
try:
|
||||
json_chunk: Final = json.loads(chunk)
|
||||
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
|
||||
return self.chunk_parser(chunk=json_chunk)
|
||||
except ValueError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -126,7 +131,7 @@ class ModelResponseIterator:
|
|||
if chunk == "[DONE]":
|
||||
raise StopAsyncIteration
|
||||
try:
|
||||
json_chunk: Final = json.loads(chunk)
|
||||
json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
|
||||
return self.chunk_parser(chunk=json_chunk)
|
||||
except ValueError as e:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ from typing import Final
|
|||
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import FileTypes
|
||||
|
||||
from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url
|
||||
from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig
|
||||
|
||||
FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth"
|
||||
SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size")
|
||||
|
|
@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
|
|||
)
|
||||
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
|
||||
"prompt": prompt,
|
||||
"image_url": to_data_url(next(iter(images))),
|
||||
"image_url": image_to_data_url(next(iter(images))),
|
||||
**provider_params,
|
||||
}
|
||||
return request_body, ()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
import base64
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import (
|
||||
map_gpt_image_quality,
|
||||
|
|
@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SeekableBinaryReader(Protocol):
|
||||
def tell(self) -> int: ...
|
||||
|
||||
def seek(self, offset: int) -> int: ...
|
||||
|
||||
def read(self) -> bytes: ...
|
||||
|
||||
|
||||
def _read_image_bytes(image: object) -> bytes:
|
||||
if isinstance(image, bytes):
|
||||
return image
|
||||
if isinstance(image, tuple):
|
||||
return _read_image_bytes(image[1])
|
||||
if isinstance(image, os.PathLike):
|
||||
return Path(image).read_bytes()
|
||||
if isinstance(image, _SeekableBinaryReader):
|
||||
position: Final = image.tell()
|
||||
image.seek(0)
|
||||
data: Final = image.read()
|
||||
image.seek(position)
|
||||
return data
|
||||
raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}")
|
||||
|
||||
|
||||
def to_data_url(image: object) -> str:
|
||||
if isinstance(image, str):
|
||||
return image
|
||||
image_bytes: Final = _read_image_bytes(image)
|
||||
mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
|
||||
return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
|
||||
|
||||
|
||||
def _first(value: object) -> object:
|
||||
return value[0] if isinstance(value, list) and value else value
|
||||
|
||||
|
|
@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig):
|
|||
raise ValueError("Fal AI image edit requires at least one input image")
|
||||
mask: Final = _first(image_edit_optional_request_params.get("mask"))
|
||||
mask_field: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
|
||||
MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({})
|
||||
)
|
||||
provider_params: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
|
||||
)
|
||||
request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict
|
||||
"prompt": prompt,
|
||||
"image_urls": tuple(to_data_url(img) for img in images),
|
||||
"image_urls": tuple(image_to_data_url(img) for img in images),
|
||||
**mask_field,
|
||||
**provider_params,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
|||
BaseTranslation,
|
||||
StreamTransformSink,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -121,7 +122,7 @@ def _parse_json_dict_or_none(payload: str) -> Mapping[str, object] | None:
|
|||
def _sse_payload_texts(sse_text: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
text
|
||||
for line in sse_text.splitlines()
|
||||
for line in split_sse_lines(sse_text)
|
||||
if line.startswith("data:")
|
||||
for payload in (line[len("data:") :].strip(),)
|
||||
if payload and payload != "[DONE]"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching import InMemoryCache
|
||||
|
|
@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
|
|||
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _GigaChatTokenResponse(TypedDict):
|
||||
tok: NotRequired[ReadOnly[str | None]]
|
||||
access_token: NotRequired[ReadOnly[str | None]]
|
||||
exp: NotRequired[ReadOnly[int | float | str | None]]
|
||||
expires_at: NotRequired[ReadOnly[int | float | str | None]]
|
||||
|
||||
|
||||
_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
|
||||
|
||||
|
||||
class GigaChatAuthError(BaseLLMException):
|
||||
"""GigaChat authentication error."""
|
||||
|
||||
|
|
@ -230,7 +243,7 @@ async def _request_token_async(
|
|||
|
||||
def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
|
||||
"""Parse OAuth token response."""
|
||||
data: Final = response.json()
|
||||
data: Final = _TOKEN_RESPONSE.validate_python(response.json())
|
||||
|
||||
# GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
|
||||
access_token: Final = data.get("tok") or data.get("access_token")
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
)
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
|
@ -357,7 +358,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
|
|||
api_key: str | None = None,
|
||||
) -> list[dict[str, str]]:
|
||||
streamed_response: Final = CustomStreamWrapper(
|
||||
completion_stream=response.iter_lines(),
|
||||
completion_stream=iter_sse_lines(response.iter_text()),
|
||||
model=model,
|
||||
custom_llm_provider="huggingface",
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
|
@ -36,12 +37,12 @@ class LangGraphSSEStreamIterator:
|
|||
|
||||
def __iter__(self):
|
||||
"""Initialize sync iteration."""
|
||||
self.line_iterator = self.response.iter_lines()
|
||||
self.line_iterator = iter_sse_lines(self.response.iter_text())
|
||||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
"""Initialize async iteration."""
|
||||
self.async_line_iterator = self.response.aiter_lines()
|
||||
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
|
||||
return self
|
||||
|
||||
def _parse_sse_line(self, line: str) -> ModelResponseStream | None:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
|
||||
from litellm.utils import EmbeddingResponse, ModelResponse, Usage
|
||||
|
||||
from ..common_utils import OobaboogaError
|
||||
|
|
@ -74,7 +75,7 @@ def completion(
|
|||
stream=optional_params["stream"] if "stream" in optional_params else False,
|
||||
)
|
||||
if "stream" in optional_params and optional_params["stream"] is True:
|
||||
return response.iter_lines()
|
||||
return iter_sse_lines(response.iter_text())
|
||||
else:
|
||||
return oobabooga_config.transform_response(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -1,31 +1,5 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
|
||||
from .dalle2_transformation import DallE2ImageEditConfig
|
||||
from .transformation import OpenAIImageEditConfig
|
||||
|
||||
__all__ = [
|
||||
"DallE2ImageEditConfig",
|
||||
"OpenAIImageEditConfig",
|
||||
"get_openai_image_edit_config",
|
||||
]
|
||||
|
||||
|
||||
def get_openai_image_edit_config(model: str) -> BaseImageEditConfig:
|
||||
"""
|
||||
Get the appropriate OpenAI image edit config based on the model.
|
||||
|
||||
Args:
|
||||
model: The model name (e.g., "dall-e-2", "gpt-image-1")
|
||||
|
||||
Returns:
|
||||
The appropriate config instance for the model
|
||||
"""
|
||||
model_normalized: Final = model.lower().replace("-", "").replace("_", "")
|
||||
|
||||
if model_normalized == "dalle2":
|
||||
return DallE2ImageEditConfig()
|
||||
else:
|
||||
# Default to standard OpenAI config for gpt-image-1 and other models
|
||||
return OpenAIImageEditConfig()
|
||||
|
|
|
|||
|
|
@ -1,98 +0,0 @@
|
|||
from io import BufferedReader
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
import litellm
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.types.images.main import ImageEditRequestParams
|
||||
from litellm.types.llms.openai import FileTypes
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from .transformation import OpenAIImageEditConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class DallE2ImageEditConfig(OpenAIImageEditConfig):
|
||||
"""
|
||||
DALL-E-2 specific configuration for image edit API.
|
||||
|
||||
DALL-E-2 only supports editing a single image (not an array).
|
||||
Uses "image" field name instead of "image[]".
|
||||
"""
|
||||
|
||||
def transform_image_edit_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str | None,
|
||||
image: FileTypes | None,
|
||||
image_edit_optional_request_params: dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> tuple[dict, RequestFiles]:
|
||||
"""
|
||||
Transform image edit request for DALL-E-2.
|
||||
|
||||
DALL-E-2 only accepts a single image with field name "image" (not "image[]").
|
||||
"""
|
||||
request_params: Final = {
|
||||
"model": model,
|
||||
**image_edit_optional_request_params,
|
||||
}
|
||||
if image is not None:
|
||||
request_params["image"] = image
|
||||
if prompt is not None:
|
||||
request_params["prompt"] = prompt
|
||||
|
||||
request: Final = ImageEditRequestParams(**request_params)
|
||||
request_dict: Final = cast(dict, request)
|
||||
|
||||
#########################################################
|
||||
# Separate images and masks as `files` and send other parameters as `data`
|
||||
#########################################################
|
||||
_image_list: Final = request_dict.get("image")
|
||||
_mask = request_dict.get("mask")
|
||||
data_without_files: Final = {k: v for k, v in request_dict.items() if k not in ["image", "mask"]}
|
||||
files_list: Final[list[tuple[str, Any]]] = []
|
||||
|
||||
# Handle image parameter - DALL-E-2 only supports single image
|
||||
if _image_list is not None:
|
||||
image_list: Final = [_image_list] if not isinstance(_image_list, list) else _image_list
|
||||
|
||||
# Validate only one image is provided
|
||||
if len(image_list) > 1:
|
||||
raise litellm.BadRequestError(
|
||||
message="DALL-E-2 only supports editing a single image. Please provide one image.",
|
||||
model=model,
|
||||
llm_provider="openai",
|
||||
)
|
||||
|
||||
# Use "image" field name (singular) for DALL-E-2
|
||||
for _image in image_list:
|
||||
if _image is not None:
|
||||
self._add_image_to_files(
|
||||
files_list=files_list,
|
||||
image=_image,
|
||||
field_name="image",
|
||||
)
|
||||
|
||||
# Handle mask parameter if provided
|
||||
if _mask is not None:
|
||||
# Handle case where mask can be a list (extract first mask)
|
||||
if isinstance(_mask, list):
|
||||
_mask = _mask[0] if _mask else None
|
||||
|
||||
if _mask is not None:
|
||||
mask_content_type: Final[str] = ImageEditRequestUtils.get_image_content_type(_mask)
|
||||
if isinstance(_mask, BufferedReader):
|
||||
files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
|
||||
else:
|
||||
files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
|
||||
|
||||
return data_without_files, files_list
|
||||
|
|
@ -96,7 +96,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
|
|||
"""
|
||||
# Build request params, only including non-None values
|
||||
request_params: Final = {
|
||||
"model": model,
|
||||
**({"model": model} if model else {}),
|
||||
**image_edit_optional_request_params,
|
||||
}
|
||||
if image is not None:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import DallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import DallE3ImageGenerationConfig
|
||||
from .gpt_transformation import GPTImageGenerationConfig
|
||||
from .guardrail_translation import (
|
||||
OpenAIImageGenerationHandler,
|
||||
|
|
@ -11,18 +5,7 @@ from .guardrail_translation import (
|
|||
)
|
||||
|
||||
__all__ = [
|
||||
"DallE2ImageGenerationConfig",
|
||||
"DallE3ImageGenerationConfig",
|
||||
"GPTImageGenerationConfig",
|
||||
"OpenAIImageGenerationHandler",
|
||||
"guardrail_translation_mappings",
|
||||
]
|
||||
|
||||
|
||||
def get_openai_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
||||
if model.startswith("dall-e-2") or model == "": # empty model is dall-e-2
|
||||
return DallE2ImageGenerationConfig()
|
||||
elif model.startswith("dall-e-3"):
|
||||
return DallE3ImageGenerationConfig()
|
||||
else:
|
||||
return GPTImageGenerationConfig()
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class DallE2ImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
OpenAI dall-e-2 image generation config
|
||||
"""
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
return ["n", "response_format", "quality", "size", "user"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
for k in non_default_params:
|
||||
if k not in optional_params:
|
||||
if k in supported_params:
|
||||
optional_params[k] = non_default_params[k]
|
||||
elif drop_params:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
response: Final = raw_response.json()
|
||||
|
||||
stringified_response: Final = response
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
original_response=stringified_response,
|
||||
)
|
||||
image_response: Final[ImageResponse] = convert_to_model_response_object(
|
||||
response_object=stringified_response,
|
||||
model_response_object=model_response,
|
||||
response_type="image_generation",
|
||||
)
|
||||
|
||||
# set optional params
|
||||
image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024
|
||||
image_response.quality = optional_params.get("quality", "standard") # always standard for dall-e-2
|
||||
image_response.output_format = optional_params.get("output_format", "png") # always png for dall-e-2
|
||||
|
||||
return image_response
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class DallE3ImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
OpenAI dall-e-3 image generation config
|
||||
"""
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
return ["n", "response_format", "quality", "size", "user", "style"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
for k in non_default_params:
|
||||
if k not in optional_params:
|
||||
if k in supported_params:
|
||||
optional_params[k] = non_default_params[k]
|
||||
elif drop_params:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
response: Final = raw_response.json()
|
||||
|
||||
stringified_response: Final = response
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
original_response=stringified_response,
|
||||
)
|
||||
image_response: Final[ImageResponse] = convert_to_model_response_object(
|
||||
response_object=stringified_response,
|
||||
model_response_object=model_response,
|
||||
response_type="image_generation",
|
||||
)
|
||||
|
||||
# set optional params
|
||||
image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024
|
||||
image_response.quality = optional_params.get("quality", "hd") # always hd for dall-e-3
|
||||
image_response.output_format = optional_params.get("output_format", "png") # always png for dall-e-3
|
||||
|
||||
return image_response
|
||||
|
|
@ -151,7 +151,7 @@ class OpenAIImageVariationsHandler:
|
|||
) -> ImageResponse:
|
||||
try:
|
||||
provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
|
||||
model=model or "", # openai defaults to dall-e-2
|
||||
model=model or "",
|
||||
provider=LlmProviders.OPENAI,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
"""
|
||||
|
||||
import ssl
|
||||
from typing import Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
from litellm._logging import redact_string, verbose_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
|
|
@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
|||
from ..common_utils import is_openai_backed_api_base
|
||||
from ..openai import OpenAIChatCompletion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import WebSocket
|
||||
|
||||
|
||||
class OpenAIRealtime(OpenAIChatCompletion):
|
||||
"""
|
||||
|
|
@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
async def async_realtime(
|
||||
self,
|
||||
model: str,
|
||||
websocket: Any,
|
||||
websocket: "WebSocket",
|
||||
logging_obj: LiteLLMLogging,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm import LlmProviders
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
|
|
@ -65,7 +66,9 @@ async def make_call(
|
|||
model_response: Final = _fake_streamed_model_response(response.json())
|
||||
completion_stream = MockResponseIterator(model_response=model_response)
|
||||
else:
|
||||
completion_stream = ModelResponseIterator(streaming_response=response.aiter_lines(), sync_stream=False)
|
||||
completion_stream = ModelResponseIterator(
|
||||
streaming_response=aiter_sse_lines(response.aiter_text()), sync_stream=False
|
||||
)
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
|
|
@ -103,7 +106,9 @@ def make_sync_call(
|
|||
model_response: Final = _fake_streamed_model_response(response.json())
|
||||
completion_stream = MockResponseIterator(model_response=model_response)
|
||||
else:
|
||||
completion_stream = ModelResponseIterator(streaming_response=response.iter_lines(), sync_stream=True)
|
||||
completion_stream = ModelResponseIterator(
|
||||
streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
|
|
|
|||
|
|
@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
|
|||
"""
|
||||
|
||||
import base64
|
||||
import builtins
|
||||
import struct
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
|
|
@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
|
|||
_INT: Final = TypeAdapter(int)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True))
|
||||
class _PerplexityEmbeddingResponse(TypedDict, total=False):
|
||||
model: ReadOnly[str | None]
|
||||
object: ReadOnly[Literal["list"]]
|
||||
data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
|
||||
usage: ReadOnly[builtins.object]
|
||||
|
||||
|
||||
_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
|
||||
|
||||
|
||||
class PerplexityEmbeddingError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
|
|||
litellm_params: dict = {},
|
||||
) -> EmbeddingResponse:
|
||||
try:
|
||||
raw_response_json: Final = raw_response.json()
|
||||
raw_response_json: Final[object] = raw_response.json()
|
||||
except Exception:
|
||||
raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
|
||||
model_response.model = raw_response_json.get("model", model)
|
||||
model_response.object = raw_response_json.get("object", "list")
|
||||
response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
|
||||
raw_response_json
|
||||
)
|
||||
|
||||
raw_data: Final = raw_response_json.get("data", [])
|
||||
model_response.model = response_data.get("model", model)
|
||||
model_response.object = response_data.get("object", "list")
|
||||
|
||||
raw_data: Final = response_data.get("data", [])
|
||||
decoded_data: Final[list[dict[str, object]]] = []
|
||||
for item in raw_data:
|
||||
decoded_item = dict(_JSON_OBJECT.validate_python(item))
|
||||
|
|
@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
|
|||
decoded_data.append(decoded_item)
|
||||
model_response.data = decoded_data
|
||||
|
||||
usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
|
||||
usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=_INT.validate_python(
|
||||
usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
|
||||
from litellm.utils import CustomStreamWrapper, ModelResponse
|
||||
|
||||
from ..common_utils import PredibaseError
|
||||
|
|
@ -34,7 +35,7 @@ async def make_call(
|
|||
if response.status_code != 200:
|
||||
raise PredibaseError(status_code=response.status_code, message=response.text)
|
||||
|
||||
completion_stream: Final = response.aiter_lines()
|
||||
completion_stream: Final = aiter_sse_lines(response.aiter_text())
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
|
|
@ -163,7 +164,7 @@ class PredibaseChatCompletion:
|
|||
timeout=timeout,
|
||||
)
|
||||
_response: Final = CustomStreamWrapper(
|
||||
response.iter_lines(),
|
||||
iter_sse_lines(response.iter_text()),
|
||||
model,
|
||||
custom_llm_provider="predibase",
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,49 @@
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
calculate_image_response_cost_from_usage,
|
||||
flat_image_cost,
|
||||
generic_cost_per_token,
|
||||
resolve_image_model_info,
|
||||
)
|
||||
from litellm.types.utils import ImageResponse, ModelInfo, Usage
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
||||
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")
|
||||
|
||||
|
||||
def _deployment_image_token_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
|
||||
if model_info is None or model_info.get("output_cost_per_image_token") is not None:
|
||||
return model_info
|
||||
output_cost_per_token: Final = model_info.get("output_cost_per_token")
|
||||
if output_cost_per_token is None:
|
||||
return model_info
|
||||
priced: Final[ModelInfo] = {**model_info, "output_cost_per_image_token": output_cost_per_token}
|
||||
return priced
|
||||
|
||||
|
||||
def _deployment_prices_per_image_only(model_info: ModelInfo | None) -> bool:
|
||||
return (
|
||||
model_info is not None
|
||||
and model_info.get("output_cost_per_image") is not None
|
||||
and model_info.get("output_cost_per_image_token") is None
|
||||
and model_info.get("output_cost_per_token") is None
|
||||
)
|
||||
|
||||
|
||||
def image_cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
|
||||
resolved_model_info: Final = resolve_image_model_info(
|
||||
model=model, custom_llm_provider="tencent", model_info=_deployment_image_token_pricing(model_info)
|
||||
)
|
||||
if _deployment_prices_per_image_only(model_info):
|
||||
return flat_image_cost(resolved_model_info, image_response)
|
||||
token_based_cost: Final = calculate_image_response_cost_from_usage(
|
||||
model=model,
|
||||
image_response=image_response,
|
||||
custom_llm_provider="tencent",
|
||||
model_info=resolved_model_info,
|
||||
)
|
||||
if token_based_cost is not None:
|
||||
return token_based_cost
|
||||
return flat_image_cost(resolved_model_info, image_response)
|
||||
|
|
|
|||
0
litellm/llms/tencent/image_edit/__init__.py
Normal file
0
litellm/llms/tencent/image_edit/__init__.py
Normal file
147
litellm/llms/tencent/image_edit/transformation.py
Normal file
147
litellm/llms/tencent/image_edit/transformation.py
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import override
|
||||
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.tencent.image_generation.transformation import (
|
||||
TOKENHUB_IMAGE_PARAMS,
|
||||
tokenhub_image_headers,
|
||||
tokenhub_image_params,
|
||||
tokenhub_image_response,
|
||||
tokenhub_image_url,
|
||||
)
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
_IMAGES: Final = TypeAdapter(tuple[object, ...])
|
||||
_FORM_BOOLEANS: Final = MappingProxyType({"true": True, "false": False})
|
||||
_OPENAI_PARAMS: Final = frozenset(TOKENHUB_IMAGE_PARAMS)
|
||||
|
||||
_FormEntries: TypeAlias = tuple[tuple[tuple[str, ...], object], ...]
|
||||
|
||||
|
||||
def _form_scalar(value: object) -> object:
|
||||
match value:
|
||||
case str() if value.lower() in _FORM_BOOLEANS:
|
||||
return _FORM_BOOLEANS[value.lower()]
|
||||
case str() if value.removeprefix("-").isdigit():
|
||||
return int(value)
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _form_path(key: str) -> tuple[str, ...]:
|
||||
root, bracket, rest = key.partition("[")
|
||||
if not bracket or not rest.endswith("]"):
|
||||
return (key,)
|
||||
return (root, *rest.removesuffix("]").split("]["))
|
||||
|
||||
|
||||
def _nested_form_value(entries: _FormEntries) -> object:
|
||||
match entries:
|
||||
case (((), value),):
|
||||
return _form_scalar(value)
|
||||
case _:
|
||||
return _nested_form_fields(tuple(entry for entry in entries if entry[0]))
|
||||
|
||||
|
||||
def _nested_form_fields(entries: _FormEntries) -> Mapping[str, object]:
|
||||
roots: Final = tuple(dict.fromkeys(path[0] for path, _ in entries))
|
||||
return {
|
||||
root: _nested_form_value(tuple((path[1:], value) for path, value in entries if path[0] == root))
|
||||
for root in roots
|
||||
}
|
||||
|
||||
|
||||
def _tokenhub_form_params(params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
tokenhub_entries: Final = tuple(
|
||||
(_form_path(key), value) for key, value in params.items() if key not in _OPENAI_PARAMS
|
||||
)
|
||||
return {
|
||||
**{key: value for key, value in params.items() if key in _OPENAI_PARAMS},
|
||||
**_nested_form_fields(tokenhub_entries),
|
||||
}
|
||||
|
||||
|
||||
class TencentImageEditConfig(BaseImageEditConfig):
|
||||
@override
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return list(TOKENHUB_IMAGE_PARAMS)
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: mirrors override contract
|
||||
return dict(tokenhub_image_params(image_edit_optional_params, model, drop_params))
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
return dict(tokenhub_image_headers(headers, api_key, model))
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> str:
|
||||
return tokenhub_image_url(api_base)
|
||||
|
||||
@override
|
||||
def use_multipart_form_data(self) -> bool:
|
||||
return False
|
||||
|
||||
@override
|
||||
def transform_image_edit_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str | None,
|
||||
image: object,
|
||||
image_edit_optional_request_params: Mapping[str, object],
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: Mapping[str, str],
|
||||
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
|
||||
images: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
|
||||
return {
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"images": tuple(image_to_data_url(img) for img in images if img is not None),
|
||||
**_tokenhub_form_params(image_edit_optional_request_params),
|
||||
}, ()
|
||||
|
||||
@override
|
||||
def transform_image_edit_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> ImageResponse:
|
||||
image_response: Final = tokenhub_image_response(raw_response)
|
||||
if image_response is None:
|
||||
raise BaseLLMException(
|
||||
status_code=raw_response.status_code,
|
||||
message=raw_response.text,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
return image_response
|
||||
0
litellm/llms/tencent/image_generation/__init__.py
Normal file
0
litellm/llms/tencent/image_generation/__init__.py
Normal file
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue