diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py index 577bb32fcf0..af4054ea92e 100644 --- a/.github/scripts/smoke_test_native_wheel.py +++ b/.github/scripts/smoke_test_native_wheel.py @@ -7,6 +7,8 @@ import zipfile from pathlib import Path from typing import Final +COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json" + CHILD_SCRIPT: Final = """ from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -34,6 +36,9 @@ else: after = module.gil_stats() if not isinstance(after.get("releases"), int): raise AssertionError(f"native module unusable after panic: {after!r}") + +if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes(): + raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout") """ @@ -55,7 +60,9 @@ def main() -> int: native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name native_path.write_bytes(archive.read(native_members[0])) - result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + result: Final = subprocess.run( + (sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False + ) if result.returncode != 0: sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index 6519b65e71b..fb86237e504 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -46,3 +46,24 @@ jobs: HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF" + - name: Check whether the pull request changes the cost map + id: changed + env: + MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then + echo "cost_map=false" >> "$GITHUB_OUTPUT" + else + echo "cost_map=true" >> "$GITHUB_OUTPUT" + fi + - if: steps.changed.outputs.cost_map == 'true' + run: rustup toolchain install --no-self-update + - name: Load the cost map through model-catalog + if: steps.changed.outputs.cost_map == 'true' + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json" + cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \ + -- "$RUNNER_TEMP/model_prices_and_context_window.json" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 89ca60ad9d1..bd7343ea222 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -376,10 +376,10 @@ jobs: exit 0 fi files="$(git diff --name-only "$base" HEAD)" - if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then + if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then echo "relevant=true" >> "$GITHUB_OUTPUT" else - echo "No proxy, types or generator changes in this pull request, nothing to verify." + echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify." echo "relevant=false" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index ff9db13bd25..6aeb66ed37c 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -12,6 +12,7 @@ on: paths: - "terraform/provider/**" - "litellm/proxy/**" + - "enterprise/**" - ".github/workflows/test-terraform-provider.yml" permissions: diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index e7865817048..509271cbf54 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -322,6 +322,7 @@ jobs: tests/unit/llms/openai_like tests/unit/llms/opencode tests/unit/llms/opensandbox + tests/unit/llms/opensearch tests/unit/llms/openrouter tests/unit/llms/ovhcloud tests/unit/llms/parallel_ai diff --git a/backend/Dockerfile b/backend/Dockerfile index 1c22f914100..aad59ba17c6 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0 COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust +COPY model_prices_and_context_window.json ./ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ diff --git a/codecov.yaml b/codecov.yaml index 5b30e78a8fb..521088868bd 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -15,9 +15,10 @@ ignore: flag_management: default_rules: carryforward: true - # Dead flags no CI job uploads anymore: their carried-forward sessions were - # measured against old revisions, and the stale line maps mark comment lines - # of since-edited files as missed, sinking patch coverage on unrelated PRs. + # Dead flags no CI job uploads anymore, and flags whose job is path-gated on + # PRs: their carried-forward sessions were measured against old revisions, and + # the stale line maps mark comment lines of since-edited files as missed, + # sinking patch coverage on unrelated PRs. individual_flags: - name: proxy-mgmt-behavior carryforward: false @@ -99,6 +100,10 @@ flag_management: carryforward: false - name: proxy-db-endpoints-and-responses carryforward: false + - name: roi-postgres + carryforward: false + - name: redis-compat + carryforward: false component_management: individual_components: diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 6dc8bab45ce..9d228627543 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0 COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust +COPY model_prices_and_context_window.json ./ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index e059d45e037..a3488d6114c 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -259,3 +259,9 @@ itself. `lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published + +## Shutdown accounting + +`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged + +Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml` diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py index e22639daaef..75c5bcf4e1e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py @@ -14,11 +14,9 @@ either, so the migration keeps today's behavior for it. """ import os -import re -from collections.abc import Callable, Iterator +from collections.abc import Callable from dataclasses import dataclass from functools import reduce -from itertools import accumulate from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal @@ -43,7 +41,7 @@ MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIME DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0 REFUSAL: Final = "Existing rows would make a pending schema change fail:" -Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"] +Kind = Literal["too_long"] @dataclass(frozen=True, slots=True) @@ -54,9 +52,6 @@ class Check: statement: str source: str index: str | None = None - referenced_table: str | None = None - referenced_columns: tuple[str, ...] = () - expression: str | None = None @dataclass(frozen=True, slots=True) @@ -100,33 +95,6 @@ class ColumnType: ColumnTypeLookup = Callable[[str, str], ColumnType | None] -_IDENT: Final = r'"?(\w+)"?' -_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?' -_CREATE_INDEX: Final = re.compile( - rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?" - rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)", - re.IGNORECASE | re.DOTALL, -) -_ALTER_TABLE: Final = re.compile( - rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL -) -_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE) -_ADD_COLUMN: Final = re.compile( - rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$", - re.IGNORECASE | re.DOTALL, -) -_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?" -_ADD_FOREIGN_KEY: Final = re.compile( - rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$", - re.IGNORECASE | re.DOTALL, -) -_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL) -_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL) -_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE) -_COMMENT: Final = re.compile(r"--[^\n]*") -_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE) -_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE) - def migration_data_check_timeout_seconds() -> float: """Seconds one count query may run before the preflight gives up on verifying it.""" @@ -145,117 +113,6 @@ def migration_data_check_timeout_seconds() -> float: return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS -def statements(migration_sql: str) -> tuple[str, ...]: - """The statements of one migration file, comments stripped, empty ones dropped.""" - parts: Final = _COMMENT.sub("", migration_sql).split(";") - return tuple(part.strip() for part in parts if part.strip()) - - -def _split_top_level(text: str) -> tuple[str, ...]: - """Split on the commas outside parentheses.""" - depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text) - cuts: Final = tuple( - position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0 - ) - starts: Final = (0, *(cut + 1 for cut in cuts)) - stops: Final = (*cuts, len(text)) - pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True)) - return tuple(piece for piece in pieces if piece) - - -def _plain_columns(column_list: str) -> tuple[str, ...] | None: - """The column names of an index or constraint column list, None when one entry is an - expression the derivation cannot turn into a count.""" - matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list)) - if any(match is None for match in matches): - return None - return tuple(match.group(1) for match in matches if match is not None) - - -def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]: - unique, name, table, method, column_list = match.groups() - columns: Final = _plain_columns(column_list) - if columns is None: - return - label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"' - if method is None or method.lower() == "btree": - yield Check("too_long", table, columns, label, source, index=name) - if unique: - yield Check("duplicates", table, columns, label, source, index=name) - - -def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]: - if (set_not_null := _SET_NOT_NULL.match(action)) is not None: - yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source) - return - if (add_column := _ADD_COLUMN.match(action)) is not None: - column, body = add_column.groups() - if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None: - yield Check( - "any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source - ) - return - if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None: - constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups() - own: Final = _plain_columns(columns) - theirs: Final = _plain_columns(referenced_columns) - if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper(): - return - yield Check( - "orphans", - table, - own, - _constraint_label(table, constraint, "FOREIGN KEY"), - source, - referenced_table=referenced_table, - referenced_columns=theirs, - ) - return - if (unique := _ADD_UNIQUE.match(action)) is not None: - constraint, columns = unique.groups() - unique_columns: Final = _plain_columns(columns) - if unique_columns is not None: - yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source) - return - if (check := _ADD_CHECK.match(action)) is not None: - constraint, expression, not_valid = check.groups() - if not_valid is None: - yield Check( - "check_violations", - table, - (), - _constraint_label(table, constraint, "CHECK"), - source, - expression=expression, - ) - - -def _constraint_label(table: str, constraint: str | None, kind: str) -> str: - named: Final = f'CONSTRAINT "{constraint}" ' if constraint else "" - return f'ALTER TABLE "{table}" ADD {named}{kind}' - - -def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]: - """The count queries one SQL statement needs before it can be applied, empty for a - statement that cannot fail on existing rows or that the derivation cannot read.""" - if (index := _CREATE_INDEX.match(statement_text)) is not None: - return tuple(_index_checks(index, statement_text, source)) - if (alter := _ALTER_TABLE.match(statement_text)) is not None: - table, actions = alter.groups() - return tuple( - check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source) - ) # comprehension-ok: flattening the per-action checks of one statement - return () - - -def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]: - return tuple( - check - for statement_text in statements(migration_sql) - for check in checks_for_statement(statement_text, f"migration {name}") - ) # comprehension-ok: flattening the per-statement checks of one migration - - def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]: """The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`.""" return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),) @@ -424,42 +281,7 @@ def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...], def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed": - from psycopg import sql - - table: Final = sql.Identifier(schema, check.table) - columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns) - match check.kind: - case "too_long": - return _entry_size_query(check, schema, types, limit_bytes) - case "duplicates": - not_null: Final = sql.SQL(" AND ").join( - sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns - ) - return sql.SQL( - "SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d" - ).format(t=table, nn=not_null, cols=columns) - case "nulls": - return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format( - t=table, col=sql.Identifier(check.columns[0]) - ) - case "any_row": - return sql.SQL("SELECT count(*) FROM {t}").format(t=table) - case "orphans": - pairs: Final = sql.SQL(" AND ").join( - sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own)) - for own, theirs in zip(check.columns, check.referenced_columns, strict=True) - ) - present: Final = sql.SQL(" AND ").join( - sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns - ) - return sql.SQL( - "SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})" - ).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs) - case "check_violations": - return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format( - t=table, - expr=sql.SQL(check.expression or "true"), # pyright: ignore[reportArgumentType] # copied from our own migration file - ) + return _entry_size_query(check, schema, types, limit_bytes) def _applies( @@ -474,15 +296,7 @@ def _applies( return False if check.index is not None and _index_is_valid(connection, schema, check.index): return False - if check.kind == "any_row": - return types is None - if types is None: - return False - if check.kind == "too_long": - return any(column.can_overflow_btree for column in types) - if check.kind == "orphans": - return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table) - return True + return types is not None and any(column.can_overflow_btree for column in types) def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str: @@ -515,21 +329,7 @@ def _run_check( rows: Final = _int(values[0], 0) if values else 0 if rows == 0: return None - match check.kind: - case "too_long": - return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes)) - case "duplicates": - return BlockedChange(check, rows, "duplicated value groups") - case "nulls": - return BlockedChange(check, rows, "NULL") - case "any_row": - return BlockedChange( - check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows" - ) - case "orphans": - return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"') - case "check_violations": - return BlockedChange(check, rows, f"failing CHECK ({check.expression})") + return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes)) def find_blockers( @@ -571,19 +371,7 @@ def find_blockers( def _remedy(check: Check) -> str: joined: Final = ", ".join(check.columns) - match check.kind: - case "too_long": - return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.' - case "duplicates": - return f'Remove or merge the duplicate "{check.table}" ({joined}) values.' - case "nulls": - return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.' - case "any_row": - return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards." - case "orphans": - return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.' - case "check_violations": - return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.' + return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.' def _describe(finding: Finding) -> str: @@ -607,30 +395,4 @@ def _describe(finding: Finding) -> str: def report(findings: tuple[Finding, ...]) -> str: """The operator-facing refusal: one block per finding, then what to do next.""" lines: Final = "\n".join(_describe(finding) for finding in findings) - return ( - f"{REFUSAL}\n{lines}\n" - "No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again." - ) - - -def pending_change_checks( - connection: "psycopg.Connection[tuple[object, ...]]", - schema: str, - migrations_dir: Path | None, - indexes: tuple[tuple[str, str, tuple[str, ...]], ...], -) -> tuple[Check, ...]: - """Every check the pending migrations and the request-log indexes need. `migrations_dir` - is None when the schema is pushed rather than migrated, so only the indexes are checked.""" - migration_checks: Final = ( - tuple( - check - for name, migration_sql in pending_migrations(connection, schema, migrations_dir) - for check in checks_for_migration(name, migration_sql) - ) # comprehension-ok: flattening the per-migration checks - if migrations_dir is not None - else () - ) - index_checks: Final = tuple( - check for table, name, columns in indexes for check in checks_for_index(table, name, columns) - ) # comprehension-ok: flattening the per-index checks - return migration_checks + index_checks + return f"{REFUSAL}\n{lines}\nNo index was built. Fix the rows and the next index build picks it up." diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py b/litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py new file mode 100644 index 00000000000..c24742ce145 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py @@ -0,0 +1,184 @@ +"""`litellm --check_migrations_against_existing_data`: run the pending migrations for real, each +inside its own transaction that is always rolled back, so Postgres itself says whether this +database's rows and schema take them. Each migration holds the locks the real upgrade will, +and releases them before the next one runs.""" + +from __future__ import annotations + +import re +import time +from collections.abc import Callable +from dataclasses import dataclass +from functools import reduce +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from litellm_proxy_extras.migration_data_checks import pending_migrations + +if TYPE_CHECKING: + import psycopg + +_CONCURRENTLY: Final = re.compile( + r"\b((?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|REINDEX\s+\w+)\s+)CONCURRENTLY\s+", re.IGNORECASE +) +UNDEFINED_SQLSTATES: Final = frozenset({"42703", "42P01", "42704", "42883"}) + +Migration = tuple[str, str] + + +@dataclass(frozen=True, slots=True) +class Applied: + name: str + seconds: float + replayed: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class Rejected: + name: str + error: str + seconds: float + + +@dataclass(frozen=True, slots=True) +class Baseline: + """A database with tables but no `_prisma_migrations`: the boot baselines it instead of replaying + history, so the dry run asks the same question the boot does. `refusal` is None when the boot + will baseline, otherwise its reason (a custom migration folder, or the schema diff).""" + + refusal: str | None + + @property + def matches(self) -> bool: + return self.refusal is None + + +@dataclass(frozen=True, slots=True) +class DryRun: + applied: tuple[Applied, ...] + rejected: Rejected | None + baseline: Baseline | None = None + + @property + def ok(self) -> bool: + return self.rejected is None and (self.baseline is None or self.baseline.matches) + + +def transactional(migration_sql: str) -> str: + """The migration as it can run inside a transaction: `CONCURRENTLY` index builds become plain ones.""" + return _CONCURRENTLY.sub(r"\1", migration_sql) + + +def _attempt( + connection: psycopg.Connection[tuple[object, ...]], + schema: str, + chain: tuple[Migration, ...], + clock: Callable[[], float], +) -> tuple[float, psycopg.Error | None]: + import psycopg + from psycopg import sql + + started: Final = clock() + try: + with connection.transaction(force_rollback=True): + connection.execute(sql.SQL("SET LOCAL search_path TO {}").format(sql.Identifier(schema))) + for _, migration_sql in chain: + connection.execute(transactional(migration_sql)) + except psycopg.Error as exc: + return clock() - started, exc + return clock() - started, None + + +def _run( + connection: psycopg.Connection[tuple[object, ...]], + schema: str, + pending: tuple[Migration, ...], + position: int, + done: DryRun, + clock: Callable[[], float], +) -> DryRun: + """One migration in its own rolled-back transaction. A migration that fails because something + an earlier pending migration creates does not exist yet is rerun with those replayed in front.""" + if done.rejected is not None: + return done + name: Final = pending[position][0] + seconds, error = _attempt(connection, schema, (pending[position],), clock) + if error is None: + return DryRun(done.applied + (Applied(name, seconds),), None, done.baseline) + if error.sqlstate not in UNDEFINED_SQLSTATES or position == 0: + return DryRun(done.applied, Rejected(name, str(error).strip(), seconds), done.baseline) + earlier: Final = pending[:position] + chained_seconds, chained_error = _attempt(connection, schema, earlier + (pending[position],), clock) + if chained_error is None: + replayed: Final = tuple(earlier_name for earlier_name, _ in earlier) + return DryRun(done.applied + (Applied(name, chained_seconds, replayed),), None, done.baseline) + return DryRun(done.applied, Rejected(name, str(chained_error).strip(), chained_seconds), done.baseline) + + +def _has_ledger(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool: + from psycopg import sql + + row: Final = connection.execute( + "SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(connection),) + ).fetchone() + return row is not None and row[0] is not None + + +def _has_litellm_tables(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool: + row: Final = connection.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE 'LiteLLM\\_%%' LIMIT 1", + (schema,), + ).fetchone() + return row is not None + + +def dry_run( + connection: psycopg.Connection[tuple[object, ...]], + schema: str, + migrations_dir: Path, + baseline_check: Callable[[], str | None] | None = None, + clock: Callable[[], float] = time.monotonic, +) -> DryRun: + """Apply every pending migration in order, each in its own transaction that is rolled back + before the next one starts. The first migration Postgres rejects ends the run with its error. + `baseline_check` (None when the boot would baseline, otherwise why it refuses) stands in for + the replay on a database that has tables but no migration history, which the boot baselines.""" + if baseline_check is not None and not _has_ledger(connection, schema) and _has_litellm_tables(connection, schema): + return DryRun((), None, Baseline(baseline_check())) + pending: Final = pending_migrations(connection, schema, migrations_dir) + return reduce( + lambda done, position: _run(connection, schema, pending, position, done, clock), + range(len(pending)), + DryRun((), None), + ) + + +def _timing(applied: Applied) -> str: + replay: Final = f" (depends on {', '.join(applied.replayed)}, replayed in front of it)" if applied.replayed else "" + return f"\n {applied.name}: {applied.seconds:.1f} s{replay}" + + +def report(result: DryRun) -> str: + """What the operator reads: the per-migration time the real upgrade will hold its locks, + or the migration Postgres rejected with Postgres's own words. Nothing was applied either way.""" + if result.baseline is not None: + if result.baseline.matches: + return ( + "No migration history and the schema matches this build, " + "the proxy baselines it at boot, nothing to dry-run." + ) + return f"No migration history and the proxy cannot baseline it at boot: {result.baseline.refusal}" + timings: Final = "".join(_timing(applied) for applied in result.applied) + if result.rejected is None: + count: Final = len(result.applied) + if count == 0: + return "No pending migrations, this database is on this version's schema." + return ( + f"{count} pending migration(s) applied one at a time in a dry run, each rolled back before the next " + f"so its locks are released, nothing changed:{timings}" + ) + return ( + f"Migration {result.rejected.name} fails on this database after {result.rejected.seconds:.1f} s, " + f"nothing was applied.\n{result.rejected.error}" + + (f"\nApplied before it in the dry run:{timings}" if timings else "") + ) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py b/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py index 5a55b35b255..1c16eb5e6b7 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py @@ -86,6 +86,18 @@ def migration_files(directory: Path) -> tuple[tuple[str, str], ...]: ) +def has_packaged_history(migrations_dir: Path) -> bool: + """Whether `migrations_dir` carries exactly the migration files and schema.prisma this package ships, + the only history the boot baselines an existing database onto.""" + packaged_dir: Final = Path(__file__).parent + migrations: Final = migration_files(migrations_dir) + return ( + bool(migrations) + and migrations == migration_files(packaged_dir) + and (migrations_dir / "schema.prisma").read_bytes() == (packaged_dir / "schema.prisma").read_bytes() + ) + + def baseline_current_schema( coordinator: MigrationCoordinator, schema: str, @@ -95,14 +107,9 @@ def baseline_current_schema( ) -> None: from psycopg import sql - packaged_dir: Final = Path(__file__).parent - migrations: Final = migration_files(migrations_dir) - if ( - not migrations - or migrations != migration_files(packaged_dir) - or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes() - ): + if not has_packaged_history(migrations_dir): raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.") + migrations: Final = migration_files(migrations_dir) coordinator.acquire_prisma_lock() existing: Final = coordinator.connection.execute( diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 528bf7def20..8b19576d86e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -11,18 +11,13 @@ from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass, replace from pathlib import Path -from typing import TYPE_CHECKING, Final, Optional, Protocol, Union +from typing import TYPE_CHECKING, Final, Protocol from litellm_proxy_extras import prisma_toolchain from litellm_proxy_extras._logging import logger from litellm_proxy_extras._redaction import _redact_credentials -from litellm_proxy_extras.migration_data_checks import ( - Finding, - find_blockers, - migration_data_check_timeout_seconds, - pending_change_checks, - report, -) +from litellm_proxy_extras.migration_dry_run import dry_run +from litellm_proxy_extras.migration_dry_run import report as dry_run_report from litellm_proxy_extras.migration_lock import held_migration_lock from litellm_proxy_extras.prisma_toolchain import ( PRISMA_COMMAND_TIMEOUT_ENV_VAR, @@ -36,7 +31,6 @@ from litellm_proxy_extras.replica_identity import ( apply_replica_identity_full, ) from litellm_proxy_extras.request_log_indexes import ( - REQUEST_LOG_INDEXES, ensure_request_log_indexes, filter_request_log_index_diff, ) @@ -52,7 +46,7 @@ class LensCheckConnect(Protocol): ) -> "psycopg.Connection[tuple[object, ...]]": ... -def str_to_bool(value: Optional[str]) -> bool: +def str_to_bool(value: str | None) -> bool: if value is None: return False return value.lower() in ("true", "1", "t", "y", "yes") @@ -169,7 +163,7 @@ def _without_sql_comments(statement: str) -> str: ).strip() -def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]: +def _without_spend_logs_pk_clauses(statement: str) -> str | None: prefix_match = _SPEND_LOGS_ALTER_RE.match(statement) if not prefix_match: return statement @@ -217,7 +211,7 @@ def _max_migration_timestamp(names) -> int: return max(_migration_timestamp(n) for n in names) -def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]: +def _redacted_command(command: object) -> str | tuple[str, ...] | list[str]: if isinstance(command, tuple): return tuple(_redact_credentials(str(argument)) for argument in command) if isinstance(command, list): @@ -404,7 +398,7 @@ class ProxyExtrasDBManager: return tuple(row) if row is not None else () @staticmethod - def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]: + def _failed_migration_logs(migration_name: str, started_at: str) -> str | None: row: Final = ProxyExtrasDBManager._read_migration_ledger( "SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz " "AND finished_at IS NULL AND rolled_back_at IS NULL", @@ -721,10 +715,10 @@ class ProxyExtrasDBManager: return row is not None @staticmethod - def _prisma_schema_param(url: str) -> Optional[str]: + def _prisma_schema_param(url: str) -> str | None: """The `schema` query param Prisma uses to pick its target schema, or None when the URL does not set one.""" - from urllib.parse import urlparse, parse_qsl + from urllib.parse import parse_qsl, urlparse return next( (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"), @@ -1424,7 +1418,6 @@ class ProxyExtrasDBManager: bool: True if setup was successful, False otherwise """ ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env()) - ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate) migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver) if not migrated: return False @@ -1433,46 +1426,69 @@ class ProxyExtrasDBManager: return True @staticmethod - def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]: - """What in the existing rows would make a pending migration or request-log index - fail, read with count queries only: no write, no table lock. Empty when nothing - blocks, when no database is configured, or when it cannot be reached, so the - migration step that follows reports the connection problem the way it does today.""" + def _baseline_refusal(prisma_dir: Path) -> str | None: + """Why `baseline_current_schema` would refuse this database at boot, None when it would baseline: + the migrations folder must be the packaged one, then `prisma migrate diff` must find the live + database and this build's schema.prisma identical.""" + from litellm_proxy_extras.migration_lock import migration_environment + from litellm_proxy_extras.migration_recovery import has_packaged_history + + if not has_packaged_history(prisma_dir): + return ( + f"{prisma_dir} is a custom migration history (LITELLM_MIGRATION_DIR), it differs from the " + "migrations this build ships, and the proxy only baselines onto the packaged history." + ) + try: + prisma_toolchain.run_prisma( + ( + _get_prisma_command(), + "migrate", + "diff", + "--from-schema-datasource", + str(prisma_dir / "schema.prisma"), + "--to-schema-datamodel", + str(prisma_dir / "schema.prisma"), + "--exit-code", + ), + timeout=prisma_toolchain.prisma_command_timeout(), + env=migration_environment(_get_prisma_env()), + ) + except subprocess.CalledProcessError as exc: + diff: Final = str(exc.stdout or exc.stderr or exc).strip() + return f"the schema differs from this build. prisma migrate diff:\n{diff}" + except subprocess.TimeoutExpired as exc: + return f"prisma migrate diff did not finish within {exc.timeout} s" + return None + + @staticmethod + def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]: + """`--check_migrations_against_existing_data`: the pending migrations applied for real on + DIRECT_URL or DATABASE_URL inside one transaction that is rolled back, so Postgres reports + what the upgrade would hit. (True, the per-migration timings) when they all apply, + (False, Postgres's own error) when one does not.""" import psycopg + if not use_migrate: + return (True, "The schema is pushed rather than migrated, there is nothing to dry-run.") database_url: Final = os.environ.get("DATABASE_URL") if not database_url: - return () + return (False, "DATABASE_URL is not set.") direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params( os.environ.get("DIRECT_URL") or database_url ) schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" - migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None - indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES) + prisma_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) try: with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection: - checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes) - return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds()) + result: Final = dry_run( + connection, + schema, + prisma_dir / "migrations", + baseline_check=lambda: ProxyExtrasDBManager._baseline_refusal(prisma_dir), + ) except psycopg.OperationalError as exc: - logger.warning( - "Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc)) - ) - return () - - @staticmethod - def migration_data_check_report(use_migrate: bool = True) -> Optional[str]: - """The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks.""" - findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate) - return report(findings) if findings else None - - @staticmethod - def raise_if_rows_block_pending_changes(use_migrate: bool) -> None: - """Refuse to run migrations that the existing rows would make fail, before any is - applied. The RuntimeError carries the report and reaches the operator through the - proxy's "Database migration cannot proceed" exit.""" - refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate) - if refusal is not None: - raise RuntimeError(refusal) + return (False, f"Could not reach the database: {_redact_credentials(str(exc))}") + return (result.ok, dry_run_report(result)) @staticmethod def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool: @@ -1724,5 +1740,4 @@ class ProxyExtrasDBManager: time.sleep(random.randrange(5, 15)) finally: os.chdir(original_dir) - pass return False diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f7d7e5f4dab..e959f77917b 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4345,6 +4345,7 @@ dependencies = [ "litellm-inference-transcription", "litellm-llms", "litellm-llms-types", + "litellm-model-catalog", "litellm-router-types", "litellm-secrets", "litellm-secrets-aws", diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs index d45078c415e..1273fd4d7f1 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs @@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[ |mapping| { mapping.error_str.contains("invalid_request_error") && !mapping.error_str.contains("Incorrect API key provided") + && !matches!(mapping.status, Some(401 | 404)) }, PublicError::BadRequest, ), @@ -132,6 +133,23 @@ mod tests { assert_eq!(classified(Some(400), text), Some(expected)); } + #[rstest::rstest] + #[case::unauthorized(Some(401), None)] + #[case::not_found(Some(404), None)] + #[case::forbidden(Some(403), Some(PublicError::BadRequest))] + #[case::rate_limited(Some(429), Some(PublicError::BadRequest))] + #[case::server_error(Some(500), Some(PublicError::BadRequest))] + #[case::no_status(None, Some(PublicError::BadRequest))] + fn invalid_request_errors_leave_401_and_404_to_the_status_mapping( + #[case] status: Option, + #[case] expected: Option, + ) { + assert_eq!( + classified(status, "invalid_request_error bad field"), + expected + ); + } + #[rstest::rstest] #[case::rate_limit_before_context_window( "rate limit and This model's maximum context length is 10", diff --git a/litellm-rust/crates/model-catalog/AGENTS.md b/litellm-rust/crates/model-catalog/AGENTS.md index fc4b28d4ef0..ff24fb81143 100644 --- a/litellm-rust/crates/model-catalog/AGENTS.md +++ b/litellm-rust/crates/model-catalog/AGENTS.md @@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust ## Tests `BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads + +`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry ` diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml index 45d16515dcd..63b9ad14768 100644 --- a/litellm-rust/crates/model-catalog/Cargo.toml +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"] [dependencies] litellm-cost = { workspace = true, features = ["serde"] } +litellm-error.workspace = true indexmap = { version = "2.14.0", features = ["serde"] } jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } @@ -24,5 +25,4 @@ time.workspace = true [dev-dependencies] jsonschema = { version = "0.55.1", default-features = false } -litellm-error.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/model-catalog/src/bin/check-registry.rs b/litellm-rust/crates/model-catalog/src/bin/check-registry.rs new file mode 100644 index 00000000000..6fdc7621a0a --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/bin/check-registry.rs @@ -0,0 +1,26 @@ +use std::{env, fs, path::PathBuf, process::ExitCode}; + +use litellm_model_catalog::Catalog; + +fn main() -> ExitCode { + let Some(path) = env::args_os().nth(1).map(PathBuf::from) else { + eprintln!("usage: check-registry "); + return ExitCode::from(2); + }; + let body = match fs::read(&path) { + Ok(body) => body, + Err(error) => { + eprintln!("{}: {error}", path.display()); + return ExitCode::FAILURE; + } + }; + let problems = Catalog::problems(&body); + for problem in &problems { + eprintln!("{}", litellm_error::Report::new(problem)); + } + if problems.is_empty() { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + } +} diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index ac5f4ffcf25..98af785b17e 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -5,7 +5,7 @@ use indexmap::IndexMap; use serde::Deserialize; use serde_json::Value; -use crate::error::{Error, Loaded, Rejects, Repr}; +use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key}; use crate::fallback::{FallbackGeneralizations, FallbackRule}; use crate::model::Model; use crate::store::{Rows, Store}; @@ -211,6 +211,36 @@ impl Catalog { }) } + pub fn problems(body: &[u8]) -> Vec { + let Loaded { + value: catalog, + errors, + } = match Self::parse(body) { + Ok(loaded) => loaded, + Err(fatal) => return vec![fatal], + }; + let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| { + EntryWire::read(entry.fields()) + .unknown + .into_iter() + .map(move |(key, _)| { + Error::new( + pointer_child(&pointer_for_key(model), &key), + Repr::UnreadField, + ) + }) + }); + let no_fallback_rules = catalog + .fallback_rules() + .is_none_or(<[_]>::is_empty) + .then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules)); + errors + .into_iter() + .chain(unread) + .chain(no_fallback_rules) + .collect() + } + pub fn row_errors(model: &str, fields: &Fields) -> Vec { let (_, _, errors) = read_row(fields); errors @@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option { ) }) } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#; + + #[rstest] + #[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])] + #[case::not_json("{".to_owned(), vec![("", "json")])] + #[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])] + #[case::unread_key( + format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#), + vec![("/m/typo_cost", "unread_field")] + )] + #[case::empty_model_key( + format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#), + vec![("//future_field", "unread_field")] + )] + #[case::unread_key_on_invalid_row( + format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#), + vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")] + )] + #[case::escaped_unread_key_on_invalid_row( + format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#), + vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")] + )] + #[case::no_rules( + r#"{"m":{"litellm_provider":"test"}}"#.to_owned(), + vec![("/fallback_generalizations", "no_fallback_rules")] + )] + #[case::empty_rules( + r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(), + vec![("/fallback_generalizations", "no_fallback_rules")] + )] + fn problems_list_what_a_cost_map_change_must_fix( + #[case] body: String, + #[case] expected: Vec<(&str, &str)>, + ) { + let problems: Vec<_> = Catalog::problems(body.as_bytes()) + .iter() + .map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind()))) + .collect(); + let expected: Vec<_> = expected + .into_iter() + .map(|(path, kind)| (path.to_owned(), kind)) + .collect(); + assert_eq!(problems, expected); + } + + #[rstest] + fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() { + let body = br#"{ + "good":{"litellm_provider":"test","custom_metadata":true}, + "bad":{"litellm_provider":"test","input_cost_per_token":"wrong"} + }"#; + let loaded = Catalog::parse(body).unwrap(); + assert_eq!(loaded.errors.len(), 1); + assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token"); + let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors); + assert_eq!(rejected.len(), 1); + assert!(catalog.lookup("bad").is_none()); + let good = catalog.lookup("good").unwrap(); + assert!(good.entry.model().is_some()); + assert_eq!( + good.entry.field("custom_metadata"), + Some(&Value::Bool(true)) + ); + assert!( + Catalog::problems(body) + .iter() + .any(|problem| problem.kind() == crate::ErrorKind::UnreadField) + ); + assert!( + Catalog::problems(body) + .iter() + .any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules) + ); + } +} diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index 425ae47687f..89e27c05f46 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -109,6 +109,12 @@ pub(crate) enum Repr { #[source] cause: Option, }, + #[error("not a field litellm reads")] + #[strum_discriminants(strum(serialize = "unread_field"))] + UnreadField, + #[error("has no rules")] + #[strum_discriminants(strum(serialize = "no_fallback_rules"))] + NoFallbackRules, } /// What a problem takes out of the served map @@ -148,7 +154,7 @@ impl Error { | Repr::NoOffPeakHours | Repr::EmptyList | Repr::InvalidDate { .. } => Rejects::Row, - Repr::MissingProvider => Rejects::Nothing, + Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing, } } } @@ -334,6 +340,8 @@ mod tests { #[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)] #[case::empty_list(Repr::EmptyList, Rejects::Row)] #[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)] + #[case::unread_field(Repr::UnreadField, Rejects::Nothing)] + #[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)] fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) { assert_eq!(Error::from(error).rejects(), expected); } diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index 30c070cfb52..31f907689d3 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() { } #[rstest] -fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() { - let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap(); - let catalog = loaded.value; - assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors); - let unrecognized: Vec<_> = catalog.unrecognized().collect(); - assert!(unrecognized.is_empty(), "{unrecognized:#?}"); - assert!( - catalog - .fallback_rules() - .is_some_and(|rules| !rules.is_empty()) - ); +fn the_bundled_registry_has_no_problems() { + let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY) + .iter() + .map(report) + .collect(); + assert!(problems.is_empty(), "{problems:#?}"); const REFERENCE_MODEL_COUNT: usize = 4303; + let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value; assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9); } diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index d90fe073b20..26858b2afe2 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -106,6 +106,18 @@ GIL handling to `litellm-host-python`. ## Data Handling +- Reuse large immutable bundled data and parsed catalogs across calls, including in + replacement APIs. Share Rust storage and cache repeated Python projections instead + of reparsing, serializing or copying the full payload on every access +- Cache owned Python objects with `PyOnceLock>`, then bind and clone references + while attached. Do not use blocking `OnceLock::get_or_init` while attached + ([PyO3 initialization](https://pyo3.rs/v0.29.2/faq.html), + [reference ownership](https://pyo3.rs/v0.29.2/types.html)) +- Never reuse Python objects across interpreters or interpreter reinitialization. + Once-only caches are for immutable data. Reloadable/custom data needs versioned + entries or invalidation, and caller-mutable Python containers must not be shared +- Test unchanged contents and allocation reuse, not just equal values. Keep this + coverage when replacing a bundled-data accessor with a catalog API - OCR payloads can contain personal data and large base64 images. Do not log payloads or provider responses. - Avoid copying large payloads more than needed. The current JSON round-trip is diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 9d69cd8c14b..a4e1c57ab1e 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -49,6 +49,7 @@ litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true litellm-host-python.workspace = true +litellm-model-catalog.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } pyo3.workspace = true prost.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12827b76ff3..683e90d02a2 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -8,6 +8,7 @@ mod execution; mod http; mod logger; mod marshal; +mod model_registry; mod preflight; mod python_settings; mod routes; @@ -30,6 +31,8 @@ mod _native { #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] + use crate::model_registry::bundled_model_cost_map; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -110,6 +113,7 @@ mod tests { "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "trace_encode_error", + "bundled_model_cost_map", "TokenCounter", "Tokenizer", "gil_stats", diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs new file mode 100644 index 00000000000..b7b7cd1fbb3 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -0,0 +1,30 @@ +use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyBytes; + +#[pyfunction] +pub fn bundled_model_cost_map(py: Python<'_>) -> Bound<'_, PyBytes> { + static BYTES: PyOnceLock> = PyOnceLock::new(); + BYTES + .get_or_init(py, || { + PyBytes::new(py, litellm_model_catalog::BUNDLED_REGISTRY).unbind() + }) + .bind(py) + .clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + fn repeated_calls_reuse_the_bundled_registry_bytes() { + Python::initialize(); + let first = Python::attach(|py| bundled_model_cost_map(py).unbind()); + Python::attach(|py| { + let second = bundled_model_cost_map(py); + assert!(first.bind(py).is(&second)); + assert_eq!(second.as_bytes(), litellm_model_catalog::BUNDLED_REGISTRY); + }); + } +} diff --git a/litellm/__init__.py b/litellm/__init__.py index 651997321a3..a1bda4aa9f7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -39,6 +39,7 @@ from litellm._logging import ( turn_on_json, log_level, ) +from litellm._logging import suppress_startup_output as suppress_startup_output import re from litellm.constants import ( DEFAULT_BATCH_SIZE, @@ -684,6 +685,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() @@ -980,6 +982,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "kyma": + kyma_models.add(key) elif value.get("litellm_provider") == "streamlake": streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": @@ -1158,6 +1162,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | kyma_models | streamlake_models | zerogpu_models | v0_models @@ -1289,6 +1294,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "kyma": kyma_models, "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 244efc8047b..c2243b90fbf 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families" _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) +_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) + @contextmanager def internal_sub_call() -> Generator[None]: @@ -123,6 +125,30 @@ def current_service_caller() -> str | None: return _service_caller.get() +@contextmanager +def mirrored_call() -> Generator[None]: + """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included.""" + token: Final = _mirrored_call.set(True) + try: + yield + finally: + _restore(_mirrored_call, token) + + +def in_mirrored_call() -> bool: + return _mirrored_call.get() + + +@contextmanager +def outside_mirrored_call() -> Generator[None]: + """Logging a mirrored call: the calls its callbacks start are not mirrors themselves.""" + token: Final = _mirrored_call.set(False) + try: + yield + finally: + _restore(_mirrored_call, token) + + @contextmanager def pinned_billing_time(moment: datetime) -> Generator[None]: """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" diff --git a/litellm/_logging.py b/litellm/_logging.py index 8974f85f517..5279cb233b4 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -10,10 +10,12 @@ import sys from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter +from types import FrameType from typing import Final, TextIO from urllib.parse import unquote from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( @@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s" _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s" _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}" +_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") def _stream_is_tty(stream: TextIO | None) -> bool: @@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool: return False -def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: - """The plain-text log format, colorized only when both streams are an interactive terminal. +def _color_enabled(stream: TextIO | None) -> bool: + """True when ANSI colors written to the stream reach an interactive terminal. Honors the NO_COLOR convention from no-color.org: color is disabled when NO_COLOR is present with a non-empty value. """ - if os.environ.get("NO_COLOR"): - return _PLAIN_LOG_FORMAT - return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT + return not os.environ.get("NO_COLOR") and _stream_is_tty(stream) + + +def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool: + """True when both streams are an interactive color terminal, the one rule for log lines and startup prints.""" + return _color_enabled(stdout) and _color_enabled(stderr) + + +def terminal_text(message: str) -> str: + """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses.""" + return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message) + + +def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: + """The plain-text log format, colorized only when both streams are an interactive terminal.""" + return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT class LevelRoutingStreamHandler(logging.StreamHandler): @@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler): super().emit(record) -def _parse_json_logs_env(value: str | None) -> bool: - """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs. +def _parse_env_flag(value: str | None) -> bool: + """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it. Matches the reader in litellm-proxy-extras/_logging.py. The previous bool(os.getenv(...)) treated any non-empty value, including "false" and "0", @@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int: return getattr(logging, log_level.upper()) -json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS")) +json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS")) +suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT")) + + +class StartupOutputSettings(TypedDict, total=False): + json_logs: ReadOnly[object] + suppress_startup_output: ReadOnly[object] + + +STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings) + + # Create a handler for the logger (you may need to adapt this based on your needs) log_level: Final = os.getenv("LITELLM_LOG", "DEBUG") numeric_level: Final[int] = resolve_log_level(log_level) @@ -743,7 +770,7 @@ class JsonFormatter(Formatter): return dt.isoformat() def format(self, record): - message_str: Final = record.getMessage() + message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage()) json_record: Final[dict[str, object]] = { "message": message_str, "level": record.levelname, @@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter): """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter. Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs - behaves the same whether or not json_logs is enabled. + behaves the same whether or not json_logs is enabled. Unless it was built with the color format, + it also drops the ANSI color codes a message carries, which a piped log would store as text. """ def format(self, record: logging.LogRecord) -> str: rendered: Final = super().format(record) - formatted: Final = rendered if _is_redacted(record) else redact_string(rendered) + uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered) + formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored) trace_id: Final = getattr(record, "trace_id", None) session_id: Final = getattr(record, "session_id", None) if not trace_id and not session_id: @@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter) verbose_proxy_stdout_logger.addFilter(_diagnostic_filter) verbose_logger.addFilter(_diagnostic_filter) -_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") - def print_startup_message(message: str) -> None: """Write a boot-time message meant for the operator's terminal. - Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of - stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger - instead, on stdout like the print whatever handlers the logging config gave that logger, with the - ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log - level, whole and unredacted as the print showed it; a blank spacer produces no record. + LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them + off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing, + keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a + container log never stores escape codes. + With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one + INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging + config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at + the caller and emitted at any log level, whole and unredacted as the print showed it; a blank + spacer produces no record. """ + if litellm.suppress_startup_output is True: + return + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO) + + +def print_startup_warning(message: str) -> None: + """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output + on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on + stdout where the print put it, so a collector that filters on the level field keeps it.""" + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING) + + +def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None: if not litellm.json_logs: - print(message) # noqa: T201 # plain mode: this is the operator-terminal print it replaces + import click + + click.echo(terminal_text(message)) return text: Final = _ANSI_ESCAPE.sub("", message).strip("\n") if not text: return - frame: Final = inspect.currentframe() - caller: Final = frame.f_back if frame is not None else None record: Final = verbose_proxy_logger.makeRecord( verbose_proxy_logger.name, - logging.INFO, + level, caller.f_code.co_filename if caller is not None else __file__, caller.f_lineno if caller is not None else 0, text, @@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None: def _startup_json_handler() -> logging.Handler: - """The proxy's JSON handler on stdout, filters included, for one startup message.""" - handler: Final = LevelRoutingStreamHandler() + """The proxy's JSON handler on stdout, filters included, for one startup message or warning.""" + handler: Final = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) handler.addFilter(_secret_filter) handler.addFilter(_correlation_filter) diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py new file mode 100644 index 00000000000..fc36699bc93 --- /dev/null +++ b/litellm/anthropic_beta_gated_fields.py @@ -0,0 +1,325 @@ +"""Body fields that an ``anthropic-beta`` value gates. + +``anthropic_beta_headers_config.json`` says which beta each provider leg +carries. When a leg drops a beta, the request fields that beta unlocks +would make the provider 400. Under ``drop_params`` the same request degrades on +that leg by dropping those fields too; without it the request goes out as sent +and the provider answers its own 400, like any unsupported param. Every helper +here is copy-on-write: the caller's containers are returned untouched when +nothing is stripped, and a shallow copy is returned otherwise, so a fallback +leg still sends the full request. +""" + +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) + +ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" +ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" + +THINKING_BLOCK_BINDING: Final = "thinking.block_binding" +THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" +SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" +SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" +SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" +SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" + +_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) + +BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,), + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,), + ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,), + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + } +) + + +def _betas_unlocking(field: str) -> frozenset[str]: + return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields) + + +_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields + } # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index +) + + +def field_is_gated(field: str, active_betas: Set[str]) -> bool: + """True when no active beta unlocks ``field``, so the field must be stripped.""" + return _FIELD_BETAS[field].isdisjoint(active_betas) + + +_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) +_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING_ADAPTER.validate_python(value, strict=True) + except ValidationError: + return None + + +def _as_sequence(value: object) -> tuple[object, ...] | None: + try: + return _SEQUENCE_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _is_tool_change_block(block: object) -> bool: + mapping: Final = _as_mapping(block) + block_type: Final = mapping.get("type") if mapping is not None else None + return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES + + +def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]: + joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta") + return frozenset(piece.strip() for piece in joined.split(",") if piece.strip()) + + +def _validated_headers(headers: object) -> Mapping[str, str]: + try: + return _HEADERS_ADAPTER.validate_python(headers) + except ValidationError: + return MappingProxyType({}) + + +def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]: + """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched + case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body.""" + body_values: Final = _as_sequence(body_betas) or () + body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str)) + return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings + + +@dataclass(frozen=True, slots=True) +class StrippedThinking: + thinking: object + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedMessages: + messages: Sequence[object] + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedRequest: + request: Mapping[str, object] + stripped: tuple[str, ...] + + +def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking: + """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated.""" + mapping: Final = _as_mapping(thinking) + if mapping is None or not drops_unsupported_params(drop_params): + return StrippedThinking(thinking=thinking, stripped=()) + binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas) + display_gated: Final = mapping.get("display") == "updates" and field_is_gated( + THINKING_DISPLAY_UPDATES, active_betas + ) + if not binding_gated and not display_gated: + return StrippedThinking(thinking=thinking, stripped=()) + gated_keys: Final = frozenset( + {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())} + ) + stripped: Final = ( + *((THINKING_BLOCK_BINDING,) if binding_gated else ()), + *((THINKING_DISPLAY_UPDATES,) if display_gated else ()), + ) + return StrippedThinking( + thinking={key: value for key, value in mapping.items() if key not in gated_keys}, + stripped=stripped, + ) + + +def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages: + """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing + when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400.""" + clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas) + output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas) + content: Final = message.get("content") + blocks: Final = _as_sequence(content) if not isinstance(content, str) else None + tool_changes_gated: Final = ( + blocks is not None + and any(_is_tool_change_block(block) for block in blocks) + and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas) + ) + if not clear_at_gated and not output_config_gated and not tool_changes_gated: + return StrippedMessages(messages=(message,), stripped=()) + kept_content: Final[object] = ( + [block for block in blocks if not _is_tool_change_block(block)] + if tool_changes_gated and blocks is not None + else content + ) + gated_keys: Final = frozenset( + {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())} + ) + rewritten: Final = { + key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys + } + stripped: Final = ( + *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()), + *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()), + *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()), + ) + if kept_content in (None, "", []) and "output_config" not in rewritten: + return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED)) + return StrippedMessages(messages=(rewritten,), stripped=stripped) + + +def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages: + mapping: Final = _as_mapping(message) + if mapping is None or mapping.get("role") != "system": + return StrippedMessages(messages=(message,), stripped=()) + return _strip_beta_gated_system_message(mapping, active_betas) + + +def _ends_on_assistant(messages: Sequence[object]) -> bool: + last: Final = _as_mapping(messages[-1]) if messages else None + return last is not None and last.get("role") == "assistant" + + +def strip_beta_gated_system_messages( + messages: Sequence[object], active_betas: Set[str], *, drop_params: object +) -> StrippedMessages: + """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same + sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation + ending on an assistant turn, which the model would read as a prefill to continue.""" + if not drops_unsupported_params(drop_params): + return StrippedMessages(messages=messages, stripped=()) + per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages) + stripped: Final = tuple( + name for result in per_message for name in result.stripped + ) # comprehension-ok: flattening per-message strip reports + if not stripped: + return StrippedMessages(messages=messages, stripped=()) + kept: Final = [ + message for result in per_message for message in result.messages + ] # comprehension-ok: flattening per-message results + if _ends_on_assistant(kept) and not _ends_on_assistant(messages): + return StrippedMessages(messages=messages, stripped=()) + return StrippedMessages(messages=kept, stripped=stripped) + + +def strip_beta_gated_request_fields( + request: Mapping[str, object], active_betas: Set[str], *, drop_params: object +) -> StrippedRequest: + """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) + messages: Final = request.get("messages") + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped) + if not stripped: + return StrippedRequest(request=request, stripped=()) + rewritten: Final = { + **request, + **({"thinking": thinking.thinking} if thinking.stripped else {}), + **({"messages": system_messages.messages} if system_messages.stripped else {}), + } + return StrippedRequest(request=rewritten, stripped=stripped) + + +@dataclass(frozen=True, slots=True) +class StrippedNativeRequest: + thinking: object + thinking_stripped: bool + messages: Sequence[object] + messages_stripped: bool + stripped: tuple[str, ...] + + +def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest: + """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists.""" + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + return StrippedNativeRequest( + thinking=params.get("thinking"), + thinking_stripped=False, + messages=message_sequence if message_sequence is not None else (), + messages_stripped=False, + stripped=(), + ) + + +def strip_beta_gated_native_request( + headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object +) -> StrippedNativeRequest: + """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta`` + list on legs that carry betas in the body, plus the body parts it may rewrite.""" + if not drops_unsupported_params(drop_params): + return unstripped_native_request(optional_params, messages) + active_betas: Final = active_beta_values(headers, body_betas) + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + return StrippedNativeRequest( + thinking=thinking.thinking, + thinking_stripped=bool(thinking.stripped), + messages=system_messages.messages, + messages_stripped=bool(system_messages.stripped), + stripped=(*thinking.stripped, *system_messages.stripped), + ) + + +def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None: + if not stripped: + return + described: Final = ", ".join( + f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})" + if field in _FIELD_BETAS + else field + for field in sorted(set(stripped)) + ) + verbose_logger.warning( + "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on " + "this request, either since this provider leg does not send that anthropic-beta (see " + "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user " + "messages.", + described, + model, + provider, + ) + + +def strip_beta_gated_request_body( + request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str +) -> Mapping[str, object]: + gated: Final = strip_beta_gated_request_fields( + request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params + ) + log_beta_gated_strips(gated.stripped, model=model, provider=provider) + return gated.request diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index d68de6c9d4d..cee5ce15469 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -248,6 +248,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": null, "web-search-2025-03-05": "web-search-2025-03-05" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fec1d8e4465..07730a0fbf8 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str: return aliases.get(provider, provider) +def has_beta_headers_config(provider: str) -> bool: + config: Final = _bundled_beta_headers_config() + aliases: Final = config.get("provider_aliases", {}) + canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider + return canonical_provider in config + + def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, @@ -397,7 +404,7 @@ def update_request_with_filtered_beta( headers: dict, request_data: dict, provider: str, -) -> tuple[dict, dict]: +) -> tuple[dict[str, str], dict[str, object]]: """ Update both headers and request body beta fields based on provider support. Modifies both dicts in place and returns them. diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..1ced13445de 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -455,17 +455,33 @@ class Cache: 3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component) """ litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {} - metadata_sources: Final[tuple[dict, ...]] = ( - kwargs.get("metadata") or {}, - kwargs.get("litellm_metadata") or {}, - litellm_params.get("metadata") or {}, - litellm_params.get("litellm_metadata") or {}, + metadata_sources: Final = cast( # cast-ok: each source is the untyped request's metadata dict or empty + "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]", + ( + kwargs.get("metadata") or {}, + kwargs.get("litellm_metadata") or {}, + litellm_params.get("metadata") or {}, + litellm_params.get("litellm_metadata") or {}, + ), ) - model_group: Final[str | None] = next( - (source["model_group"] for source in metadata_sources if source.get("model_group")), None + metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources + call_model_info: Final[object] = kwargs.get("model_info") + routed_source: Final = next( + ( + source + for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata) + if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info + ), + None, ) + model_group: Final = ( + str(routed_source["model_group"]) + if routed_source is not None + else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None) + ) + caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources caching_group: Final = next( - (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None + (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None ) model_id_suffix: Final = ( f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else "" diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 924506dfa45..560a5ba06b8 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -13,7 +13,7 @@ import asyncio import json import os from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Protocol from pydantic import TypeAdapter @@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # create an embedding for prompt - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) @@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # convert to embedding - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index b892e4c3a1d..7268511fe7e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -14,7 +14,9 @@ import asyncio import json import os from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast + +from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import print_verbose, verbose_logger @@ -38,6 +40,11 @@ if TYPE_CHECKING: from litellm.router import Router +_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter( + str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + class RedisSemanticCache(BaseCache): """ Redis-backed semantic cache for LLM responses. @@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache): ) return embedding_response["data"][0]["embedding"] - def _get_cache_logic(self, cached_response: Any) -> object: + def _get_cache_logic(self, cached_response: str | bytes | None) -> object: """ Process the cached response to prepare it for use. @@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache): The processed cache response, or None if input was None """ if cached_response is None: - return cached_response + return None # Convert bytes to string if needed - if isinstance(cached_response, bytes): - cached_response = cached_response.decode("utf-8") + response_text: Final = ( + cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response + ) # Convert string representation to Python object try: - cached_response = json.loads(cached_response) + return json.loads(response_text) except json.JSONDecodeError: try: - cached_response = ast.literal_eval(cached_response) + return ast.literal_eval(response_text) except (ValueError, SyntaxError) as e: print_verbose(f"Error parsing cached response: {e}") return None - return cached_response - def set_cache(self, key: str, value: object, **kwargs) -> None: """ Store a value in the semantic cache. @@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity @@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6173794a210..580e55273f6 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -36,6 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) @@ -61,7 +62,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, - PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -631,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) - - -def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - try: - return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - - def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -1679,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1692,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) elif original_type == "input_audio": converted = with_prompt_cache_breakpoint( {"type": "input_audio", "input_audio": item.get("input_audio")}, - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: input_audio -> %s", converted) @@ -1715,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1739,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) diff --git a/litellm/constants.py b/litellm/constants.py index 42ad624026c..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_ DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range( - "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400 + "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400 ) # Maximum wall-clock seconds a streaming response is allowed to run. @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5dc0ce13de..a4a9a5bee39 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_prompt_tokens_details, select_cost_metric_for_model, ) +from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -1396,9 +1397,9 @@ def _apply_cost_margin( def _store_cost_breakdown_in_logging_obj( litellm_logging_obj: LitellmLoggingObject | None, - prompt_tokens_cost_usd_dollar: float, - completion_tokens_cost_usd_dollar: float, - cost_for_built_in_tools_cost_usd_dollar: float, + prompt_tokens_cost_usd_dollar: float | None, + completion_tokens_cost_usd_dollar: float | None, + cost_for_built_in_tools_cost_usd_dollar: float | None, total_cost_usd_dollar: float, additional_costs: dict | None = None, original_cost: float | None = None, @@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost( return float(stamped) if total_time: return total_time - if litellm_logging_obj is not None: - return litellm_logging_obj.get_response_ms() - return 0.0 + if litellm_logging_obj is None: + return 0.0 + reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj) + if reported_ms is not None and litellm_logging_obj.stream is not True: + return reported_ms + return litellm_logging_obj.get_response_ms() + + +def _adjusted_provider_billed_cost( + reported_cost: float, + custom_llm_provider: str | None, + model: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, + service_tier: str | None, + data_residency: str | None, +) -> float: + discounted_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin( + base_cost=discounted_cost, custom_llm_provider=custom_llm_provider + ) + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=None, + completion_tokens_cost_usd_dollar=None, + cost_for_built_in_tools_cost_usd_dollar=None, + total_cost_usd_dollar=adjusted_cost, + original_cost=reported_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + return adjusted_cost def completion_cost( @@ -1616,6 +1653,9 @@ def completion_cost( ) explicit_pricing: Final = custom_pricing is True or base_model is not None + caller_set_prices: Final = ( + explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None @@ -1666,6 +1706,7 @@ def completion_cost( raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) + priced_response: Final[object] = completion_response for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names): model = candidate_model try: @@ -1984,6 +2025,19 @@ def completion_cost( total_time=total_time, ) model_for_discount = custom_priced_request_model or model + if ( + not caller_set_prices + and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None + ): + return _adjusted_provider_billed_cost( + reported_cost=reported_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" diff --git a/litellm/exceptions.py b/litellm/exceptions.py index cd5139f5f79..28211b3dab2 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -139,6 +139,7 @@ class AuthenticationError(openai.AuthenticationError): litellm_debug_info: str | None = None, max_retries: int | None = None, num_retries: int | None = None, + body: Mapping[str, object] | None = None, ): self.status_code = 401 self.message = f"litellm.AuthenticationError: {message}" @@ -152,7 +153,7 @@ class AuthenticationError(openai.AuthenticationError): request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -189,6 +190,7 @@ class NotFoundError(openai.NotFoundError): litellm_debug_info: str | None = None, max_retries: int | None = None, num_retries: int | None = None, + body: Mapping[str, object] | None = None, ): self.status_code = 404 self.message = f"litellm.NotFoundError: {message}" @@ -202,7 +204,7 @@ class NotFoundError(openai.NotFoundError): request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -321,6 +323,17 @@ class UnsupportedEndpointError(BadRequestError): pass +class EndpointNotSupportedByModelError(BadRequestError): + def __init__(self, message: str, model: str, llm_provider: str = "") -> None: + super().__init__( # pyright: ignore[reportUnknownMemberType] # inherited public exception constructor is untyped + message=message, model=model, llm_provider=llm_provider + ) + self.message = message + self.type = "invalid_request_error" + self.param = "model" + self.code = "400" + + class UnprocessableEntityError(openai.UnprocessableEntityError): def __init__( self, diff --git a/litellm/images/main.py b/litellm/images/main.py index 6e77cbb7046..4aad5210028 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -68,6 +68,11 @@ from litellm.utils import ( ) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) +_PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE: Final = ( + LlmProviders.DASHSCOPE, + LlmProviders.QWENCLOUD, + LlmProviders.QWEN_AI_PLATFORM, +) # Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls _ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None @@ -271,6 +276,7 @@ def image_generation( """ try: args: Final = locals() + requested_api_base: Final = api_base aimg_generation: Final = kwargs.get("aimg_generation", False) litellm_call_id: Final = kwargs.get("litellm_call_id", None) logger_fn: Final = kwargs.get("logger_fn", None) @@ -468,7 +474,11 @@ def image_generation( raise ValueError(f"image generation config is not supported for {custom_llm_provider}") # Resolve api_base from litellm.api_base if not explicitly provided - _api_base: Final = api_base or litellm.api_base + _api_base: Final = ( + requested_api_base + if custom_llm_provider in _PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE + else api_base or litellm.api_base + ) litellm_params_dict["api_base"] = _api_base return llm_http_handler.image_generation_handler( @@ -641,6 +651,8 @@ def image_generation( timeout=timeout, client=custom_client, ) + elif custom_llm_provider == LlmProviders.KYMA.value: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return model_response except Exception as e: diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..fc1f1830129 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") +_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None: def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True + + +def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: hosted_flag: Final = ( None if custom_llm_provider is None @@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: return model_map_flag - version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) - if version_match is None: + fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) + if fine_tuned_base is not None: + return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None + return _named_model_breakpoint_support(model) + + +def _named_model_breakpoint_support(model: str) -> bool | None: + bare_model: Final = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_PATTERN.match(bare_model) is not None: return False + version_match: Final = _GPT_VERSION_PATTERN.match(bare_model) + if version_match is None: + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a4de1be1842..0579eb15baf 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -513,7 +513,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac data: dict, user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, - ) -> Any: + ) -> object: pass async def async_post_call_streaming_hook( diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 1a81fa019dc..d4b75fc0bce 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger): # see _RESERVED_TAG_KEYS for the rationale. if self.cost_tag_keys: allow: Final = set(self.cost_tag_keys) + client_allow: Final = allow - {"is_silent_experiment"} for rt in log.get("request_tags") or []: if not isinstance(rt, str) or ":" not in rt: continue k, _, v = rt.partition(":") - if k in allow and v: + if k in client_allow and v: self._set_custom_tag(tags, k, v) for k, v in metadata.items(): if k in allow and v is not None and not isinstance(v, (dict, list)): @@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): nested = metadata.get(nested_key) if isinstance(nested, dict): for k, v in nested.items(): - if k in allow and v is not None and not isinstance(v, (dict, list)): + if k in client_allow and v is not None and not isinstance(v, (dict, list)): self._set_custom_tag(tags, k, str(v)) return tags diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..bd68ffe8cd6 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -833,6 +833,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py index 32cf93b7937..97a970c8966 100644 --- a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py +++ b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py @@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC): pass -def cast_as_primitive_value_type(value) -> str | bool | int | float: +def cast_as_primitive_value_type(value: object) -> str | bool | int | float: """ Converts a value to an OTEL-supported primitive for Arize/Phoenix observability. """ diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index bc8344fa686..e62b99f8b0f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1173,8 +1173,8 @@ class PrometheusLogger(CustomLogger): # Build label filters from valid configurations label_filters: Final = self._build_label_filters(parsed_configs) - # Pretty print the processed configuration - self._pretty_print_prometheus_config(label_filters) + if litellm.suppress_startup_output is not True: + self._pretty_print_prometheus_config(label_filters) return label_filters def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]: @@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload( **(requester_metadata if isinstance(requester_metadata, dict) else {}), **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}), **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), + "is_silent_experiment": ( + "true" + if "is_silent_experiment" in standard_logging_metadata + and standard_logging_metadata["is_silent_experiment"] is True + else None + ), } diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..c25fcfe3e70 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict): websearch_interception_params: WebSearchInterceptionConfig +def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str: + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {} + } + recorded_model: Final = agentic_view["agentic_loop_params"].get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + return qualify_provider_stripped_model( + recorded_model if isinstance(recorded_model, str) and recorded_model else model, + custom_llm_provider if isinstance(custom_llm_provider, str) else "", + ) + + class _SearchToolLitellmParams(TypedDict, total=False): search_provider: ReadOnly[str | None] @@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, response_format=response_format, ) return AgenticLoopPlan( @@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, ) return AgenticLoopPlan( run_agentic_loop=True, @@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: list[dict], optional_params: dict, kwargs: dict, + logging_obj: "LiteLLMLoggingObj | None", ) -> AgenticLoopRequestPatch: """Execute litellm.asearch() and build a Responses API rerun patch.""" search_tasks: Final = [ @@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=dict[str, object](optional_params), kwargs=dict[str, object](kwargs), + logging_obj=logging_obj, response_format=response_format, ) if request_patch.messages is None: @@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], + logging_obj: "LiteLLMLoggingObj | None", response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index d84f0b5ffbd..f95b5611831 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=custom_llm_provider, ) async def async_interactions_api_handler( @@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=responses_request.get("custom_llm_provider"), ) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index ade33831cdc..2e9b62dcfe2 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -12,6 +12,7 @@ from typing import Final, cast from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import billed_usage_cost from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig: def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, model: str | None = None, + custom_llm_provider: str | None = None, ) -> InteractionsAPIResponse: """ Transform a Responses API response to an Interactions API response. @@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: + reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), + **({} if reported_cost is None else {"cost": reported_cost}), } # Add updated (same as created for now) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 55025508e40..d940baf8b56 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType @@ -34,6 +35,49 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider + + +def _leads_with_provider(model: str) -> bool: + leading_segment, separator, _ = model.partition("/") + return bool(separator) and names_a_provider(leading_segment) + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + if _leads_with_provider(patch_model): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. @@ -973,6 +1017,16 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) +USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) + + +def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: + if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + return None + cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None + + _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 703550dd2b5..64a95eb5f28 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -413,6 +413,25 @@ def _map_openai_exception( body=getattr(original_exception, "body", None), ) elif "invalid_request_error" in error_str and "Incorrect API key provided" not in error_str: + invalid_request_status: Final = getattr(original_exception, "status_code", None) + if invalid_request_status == 401: + raise AuthenticationError( + message=f"{exception_provider} - {message}", + llm_provider=custom_llm_provider, + model=model, + response=response, + litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), + ) + if invalid_request_status == 404: + raise NotFoundError( + message=f"{exception_provider} - {message}", + model=model, + llm_provider=custom_llm_provider, + response=response, + litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), + ) raise BadRequestError( message=f"{exception_provider} - {message}", llm_provider=custom_llm_provider, diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index b19bcd941fa..940730eeb9f 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -31,7 +31,13 @@ EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = Mapping ) -def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None: +def native_health_check_mode( + model: str, custom_llm_provider: str | None +) -> Literal["anthropic_messages", "realtime"] | None: + if custom_llm_provider == "soniox": + from litellm.llms.soniox.common_utils import soniox_health_check_mode + + return soniox_health_check_mode(model) if custom_llm_provider != "bedrock_mantle": return None from litellm.llms.bedrock_mantle.common_utils import mantle_health_check_mode diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..c0e988412a1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -23,7 +23,7 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import post_response_phase +from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase from litellm._logging import ( is_debugging_on, redact_string, @@ -162,9 +162,10 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, + AgenticLoopParams, CachingDetails, CallTypes, CostBreakdown, @@ -314,7 +315,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -808,6 +809,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment: Final = in_mirrored_call() # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1158,6 +1160,9 @@ class Logging(LiteLLMLoggingBaseClass): if "custom_llm_provider" in self.model_call_details: self.custom_llm_provider = self.model_call_details["custom_llm_provider"] + def record_agentic_loop_params(self, params: AgenticLoopParams) -> None: + self.model_call_details["agentic_loop_params"] = params + def record_applied_timeout(self, timeout: float | Timeout) -> None: self.applied_timeout = timeout self.litellm_params["timeout"] = timeout @@ -1937,10 +1942,10 @@ class Logging(LiteLLMLoggingBaseClass): def set_cost_breakdown( self, - input_cost: float, - output_cost: float, + input_cost: float | None, + output_cost: float | None, total_cost: float, - cost_for_built_in_tools_cost_usd_dollar: float, + cost_for_built_in_tools_cost_usd_dollar: float | None, additional_costs: dict | None = None, original_cost: float | None = None, discount_percent: float | None = None, @@ -1979,14 +1984,17 @@ class Logging(LiteLLMLoggingBaseClass): self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( - input_cost=input_cost, - output_cost=output_cost, total_cost=total_cost, - tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, ) + if input_cost is not None: + self.cost_breakdown["input_cost"] = input_cost + if output_cost is not None: + self.cost_breakdown["output_cost"] = output_cost + if cost_for_built_in_tools_cost_usd_dollar is not None: + self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost if cache_creation_cost is not None and cache_creation_cost > 0: @@ -3249,7 +3257,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return self._success_handler_body( result=result, start_time=start_time, @@ -3749,7 +3757,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return await self._async_success_handler_body( result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs ) @@ -4266,13 +4274,14 @@ class Logging(LiteLLMLoggingBaseClass): ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: - return self._failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - for_async_request=for_async_request, - ) + with outside_mirrored_call(): + return self._failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + for_async_request=for_async_request, + ) finally: self._restore_correlation_context() @@ -4465,12 +4474,13 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own failure logging (including any nested calls its callbacks trigger) is fully done.""" try: - return await self._async_failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - ) + with outside_mirrored_call(): + return await self._async_failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + ) finally: self._restore_correlation_context() @@ -6559,6 +6569,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6632,6 +6643,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7432,6 +7444,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7716,9 +7729,8 @@ def get_standard_logging_metadata( ) if isinstance(metadata, dict): # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields - for key in StandardLoggingMetadata.__annotations__: - if key in metadata: - clean_metadata[key] = metadata[key] + for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}: + clean_metadata[key] = metadata[key] if metadata.get("user_api_key") is not None: if is_valid_sha256_hash(str(metadata.get("user_api_key"))): diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8b94c34942c..de911beabdd 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.databricks import DatabricksTool from litellm.types.llms.openai import ( + ChatCompletionAnnotation, ChatCompletionReasoningItem, ChatCompletionThinkingBlock, ImageURLListItem, @@ -59,6 +60,31 @@ _CHOICES_FIELDS: Final[frozenset] = frozenset(Choices.model_fields.keys()) _MODEL_RESPONSE_FIELDS: Final[frozenset] = frozenset(ModelResponse.model_fields.keys()) | {"usage"} +_ANNOTATION_ENTRIES: Final = TypeAdapter(tuple[object, ...]) + + +def _is_provider_specific_annotation(annotation: object) -> bool: + match annotation: + case {"type": "url_citation"}: + return False + case {"type": _}: + return True + case _: + return False + + +def _annotation_entries(annotations: object) -> tuple[object, ...]: + return _ANNOTATION_ENTRIES.validate_python(annotations) if isinstance(annotations, list) else () + + +def partition_annotations(annotations: object) -> tuple[object, tuple[object, ...]]: + entries: Final = _annotation_entries(annotations) + others: Final = tuple(annotation for annotation in entries if _is_provider_specific_annotation(annotation)) + if not others: + return annotations, () + return [annotation for annotation in entries if not _is_provider_specific_annotation(annotation)] or None, others + + def _normalize_images_for_message( images: list[dict] | None, ) -> list[ImageURLListItem] | None: @@ -787,6 +813,9 @@ def convert_to_model_response_object( provider_specific_fields = dict(choice["message"].get("provider_specific_fields", None) or {}) for f in choice["message"].keys() - _MESSAGE_FIELDS: provider_specific_fields[f] = choice["message"][f] + annotations, other_annotations = partition_annotations(choice["message"].get("annotations", None)) + if other_annotations: + provider_specific_fields["annotations"] = list(other_annotations) # Handle reasoning models that display `reasoning_content` within `content` reasoning_content, content = extract_reasoning_content(choice["message"]) @@ -809,7 +838,9 @@ def convert_to_model_response_object( reasoning_content=reasoning_content, thinking_blocks=thinking_blocks, reasoning_items=choice["message"].get("reasoning_items", None), - annotations=choice["message"].get("annotations", None), + annotations=cast( # cast-ok: provider annotation dicts pass through whole, keeping keys such as url_citation.content + list[ChatCompletionAnnotation] | None, annotations + ), images=_normalize_images_for_message(choice["message"].get("images", None)), ) finish_reason = choice.get("finish_reason", None) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..051a8fdf256 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -27,6 +27,15 @@ def _timing_window_start( return start_time, False +def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None: + reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms") + if reported_ms is None: + return None + logging_start: Final = logging_obj.start_time + window_start: Final = _timing_window_start(logging_start, logging_obj)[0] + return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000 + + def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None: if not isinstance(windows, (list, tuple)): return None diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index ff649953e69..8f217b1ea5e 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -6,7 +6,7 @@ import atexit import contextvars import inspect import logging -from collections.abc import Coroutine, Iterator +from collections.abc import Awaitable, Callable, Coroutine, Iterator from typing import Final from typing_extensions import TypedDict @@ -66,14 +66,19 @@ class LoggingWorker: max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE, concurrency: int = LOGGING_WORKER_CONCURRENCY, timeout_summary_window: float = LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS, + *, + retry_sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, ): self.timeout = timeout self.max_queue_size = max_queue_size self.concurrency = concurrency self.timeout_summary_window = timeout_summary_window self._queue: asyncio.Queue[LoggingTask] | None = None - self._worker_task: asyncio.Task | None = None - self._running_tasks: set[asyncio.Task] = set() + self._worker_task: asyncio.Task[None] | None = None + self._running_tasks: set[asyncio.Task[None]] = set() + self._overflow_tasks: set[asyncio.Task[None]] = set() # mutable-ok: done callbacks discard refs + self._pending_enqueues: dict[int, LoggingTask] = {} # mutable-ok: retries and shutdown share callback ownership + self._retry_sleep = retry_sleep self._dequeued_tasks: dict[int, LoggingTask] = {} # mutable-ok: refs so flush can rescue never-started tasks self._sem: asyncio.Semaphore | None = None self._bound_loop: asyncio.AbstractEventLoop | None = None @@ -168,6 +173,7 @@ class LoggingWorker: self._sem = None self._worker_task = None self._running_tasks.clear() + self._overflow_tasks.clear() self._timeout_summary_task = None self._timeout_burst_count = 0 self._queue = new_queue @@ -185,6 +191,10 @@ class LoggingWorker: self._sem = asyncio.Semaphore(self.concurrency) if self._worker_task is None or self._worker_task.done(): self._worker_task = asyncio.create_task(self._worker_loop()) + if not self._overflow_tasks: + self._aggressive_clear_in_progress = False + for pending in tuple(self._pending_enqueues.values()): + self._schedule_delayed_enqueue_retry(pending) async def _process_log_task( self, task: LoggingTask, sem: asyncio.Semaphore, queue: "asyncio.Queue[LoggingTask]" @@ -278,12 +288,24 @@ class LoggingWorker: # Capture the current context when enqueueing task: Final = LoggingTask(coroutine=coroutine, context=contextvars.copy_context()) + self._enqueue_task(task) + + def _enqueue_task(self, task: LoggingTask) -> None: + if self._queue is None: + return try: self._queue.put_nowait(task) except asyncio.QueueFull: - # Queue is full - handle it appropriately verbose_logger.exception("LoggingWorker queue is full") + self._pending_enqueues[id(task)] = task self._handle_queue_full(task) + else: + self._pending_enqueues.pop(id(task), None) + + def _start_overflow_task(self, coroutine: Coroutine[object, object, None]) -> None: + task: Final = asyncio.create_task(coroutine) + self._overflow_tasks.add(task) + task.add_done_callback(self._overflow_tasks.discard) def _should_start_aggressive_clear(self) -> bool: """ @@ -326,7 +348,7 @@ class LoggingWorker: if self._should_start_aggressive_clear(): self._mark_aggressive_clear_started() # Schedule clearing as async task so enqueue returns immediately (non-blocking) - asyncio.create_task(self._aggressively_clear_queue_async(task)) + self._start_overflow_task(self._aggressively_clear_queue_async(task)) else: # Cooldown active or clear in progress, schedule a delayed retry self._schedule_delayed_enqueue_retry(task) @@ -363,7 +385,7 @@ class LoggingWorker: delay: Final = self._calculate_retry_delay() # Schedule the retry as a background task - asyncio.create_task(self._retry_enqueue_task(task, delay)) + self._start_overflow_task(self._retry_enqueue_task(task, delay)) except RuntimeError: # No event loop, drop the task as we can't schedule a retry pass @@ -373,17 +395,8 @@ class LoggingWorker: Retry enqueueing the task after delay, preserving original context. This is called as a background task from _schedule_delayed_enqueue_retry. """ - await asyncio.sleep(delay) - - # Try to enqueue the task directly, preserving its original context - if self._queue is None: - return - - try: - self._queue.put_nowait(task) - except asyncio.QueueFull: - # Still full - handle it appropriately (clear or retry again) - self._handle_queue_full(task) + await self._retry_sleep(delay) + self._enqueue_task(task) def _extract_tasks_from_queue(self) -> list[LoggingTask]: """ @@ -423,15 +436,10 @@ class LoggingWorker: return extracted_tasks: Final = self._extract_tasks_from_queue() - - # Add new task to extracted tasks to process directly if new_task is not None: - self._track_dequeued(new_task) - extracted_tasks.append(new_task) - - # Process extracted tasks directly + self._enqueue_task(new_task) if extracted_tasks: - await self._process_extracted_tasks(extracted_tasks) + await asyncio.gather(*(self._process_single_task(task) for task in extracted_tasks)) except Exception as e: verbose_logger.exception("LoggingWorker error during aggressive clear: %s", e) finally: @@ -439,7 +447,7 @@ class LoggingWorker: self._aggressive_clear_in_progress = False async def _process_single_task(self, task: LoggingTask) -> None: - """Process a single task and mark it done.""" + """Process a task extracted from the queue and mark it done.""" if self._queue is None: return @@ -455,17 +463,6 @@ class LoggingWorker: self._untrack_dequeued(task) self._queue.task_done() - async def _process_extracted_tasks(self, tasks: list[LoggingTask]) -> None: - """ - Process tasks that were extracted from the queue to make room. - Processes them concurrently without semaphore limits for maximum speed. - """ - if not tasks or self._queue is None: - return - - # Process all tasks concurrently for maximum speed - await asyncio.gather(*[self._process_single_task(task) for task in tasks]) - def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, None, None]) -> None: """ Ensure the logging worker is initialized and enqueue the coroutine. @@ -473,32 +470,63 @@ class LoggingWorker: self.start() self.enqueue(async_coroutine) - async def stop(self) -> None: + @property + def pending_count(self) -> int: + return ( + (self._queue.qsize() if self._queue is not None else 0) + + len(self._dequeued_tasks) + + len(self._pending_enqueues) + ) + + async def stop(self, *, discard_pending: bool = False, timeout: float | None = None) -> None: """Stop the logging worker and clean up resources.""" + if discard_pending and self._queue is not None: + discarded: Final = self._drain_pending(self._queue) + for task in discarded: + task["coroutine"].close() + self._queue.task_done() + if discarded: + verbose_logger.warning("LoggingWorker shutdown: %d queued callbacks discarded", len(discarded)) if self._timeout_summary_task is not None: self._timeout_summary_task.cancel() self._timeout_summary_task = None self._emit_timeout_summary() - if self._worker_task is None and not self._running_tasks: - # No worker launched and no in-flight tasks to drain. - return + if discard_pending: + rejected: Final = tuple(self._pending_enqueues.values()) + self._pending_enqueues.clear() + for task in rejected: + task["coroutine"].close() + if rejected: + verbose_logger.warning("LoggingWorker shutdown: %d overflow callbacks discarded", len(rejected)) - tasks_to_cancel: Final[list[asyncio.Task]] = list(self._running_tasks) - if self._worker_task: - # Include the main worker loop so it stops fetching work. - tasks_to_cancel.append(self._worker_task) + tasks_to_cancel: Final = ( + *self._running_tasks, + *self._overflow_tasks, + *((self._worker_task,) if self._worker_task is not None else ()), + ) + if not tasks_to_cancel: + return for task in tasks_to_cancel: # Propagate cancellation to every pending task. task.cancel() # Wait for cancellation to settle; ignore errors raised during shutdown. - await asyncio.gather(*tasks_to_cancel, return_exceptions=True) + done, pending = await asyncio.wait(tasks_to_cancel, timeout=timeout) + await asyncio.gather(*done, return_exceptions=True) + if pending: + verbose_logger.warning("LoggingWorker shutdown: %d tasks still alive after cancellation", len(pending)) - self._worker_task = None + if discard_pending and self._queue is not None: + for task in self._unstarted_dequeued_tasks(): + task["coroutine"].close() + self._untrack_dequeued(task) + self._queue.task_done() + if self._worker_task is not None and self._worker_task.done(): + self._worker_task = None # Drop references to completed tasks so we can restart cleanly. - self._running_tasks.clear() + self._running_tasks.difference_update(done) async def flush(self) -> None: """Flush the logging queue. @@ -516,6 +544,9 @@ class LoggingWorker: return self.start() await self._queue.join() + while self._overflow_tasks: + await asyncio.wait(tuple(self._overflow_tasks)) + await self._queue.join() async def clear_queue(self): """ diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 101f069c0f8..b3f89b33d1d 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -17,7 +17,7 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger @@ -424,6 +424,109 @@ def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> return None +@dataclass(frozen=True, slots=True) +class InlineDocument: + filename: str + file_data: str + + +def _document_media_type(source: object) -> str: + match source: + case {"media_type": str() as media_type} if media_type: + return media_type + case _: + return "application/pdf" + + +def anthropic_inline_document(document: Mapping[str, object]) -> InlineDocument | None: + source: Final = document.get("source") + match source: + case {"type": "base64", "data": str() as data} if data: + title: Final = document.get("title") + return InlineDocument( + filename=title if isinstance(title, str) and title else "document.pdf", + file_data=f"data:{_document_media_type(source)};base64,{data}", + ) + case _: + return None + + +def is_pdf_file_part(part: object) -> bool: + match part: + case {"type": "file", "file": {"file_data": str() as file_data}} if file_data.startswith("data:"): + return file_data.startswith("data:application/pdf") + case {"type": "file", "file": {"format": str() as file_format}}: + return file_format == "application/pdf" + case {"type": "file", "file": {"filename": str() as filename}}: + return filename.lower().endswith(".pdf") + case _: + return False + + +_CONTENT_PARTS: Final = TypeAdapter(tuple[object, ...]) + + +def content_parts(message: Mapping[str, object]) -> Sequence[object]: + content: Final = message.get("content") + return _CONTENT_PARTS.validate_python(content) if isinstance(content, list) else () + + +def _has_pdf_file_part(message: Mapping[str, object]) -> bool: + return any(is_pdf_file_part(part) for part in content_parts(message)) + + +def messages_have_pdf_file_part(messages: Sequence[Mapping[str, object]]) -> bool: + return any(_has_pdf_file_part(message) for message in messages) + + +def _message_without_pdf_file_parts(message: AllMessageValues) -> AllMessageValues: + if not _has_pdf_file_part(message): + return message + kept: Final = [part for part in content_parts(message) if not is_pdf_file_part(part)] + return cast(AllMessageValues, {**message, "content": kept}) # cast-ok: same message with fewer content parts + + +def _is_pdf_only_user_turn(message: AllMessageValues) -> bool: + parts: Final = content_parts(message) + return message["role"] == "user" and bool(parts) and all(is_pdf_file_part(part) for part in parts) + + +def messages_without_pdf_file_parts(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]: + return tuple( + _message_without_pdf_file_parts(message) for message in messages if not _is_pdf_only_user_turn(message) + ) + + +_CONTENT_PART: Final = TypeAdapter(Mapping[str, object]) + + +def _file_part_as_image_url(part: object) -> object: + match part: + case {"type": "file", "file": {"file_data": str() as file_data}} if file_data: + fields: Final = _CONTENT_PART.validate_python(part) + return {key: value for key, value in fields.items() if key != "file"} | { + "type": "image_url", + "image_url": {"url": file_data}, + } + case _: + return part + + +def message_with_file_parts_as_image_urls(message: Mapping[str, object]) -> Mapping[str, object]: + parts: Final = content_parts(message) + if not parts: + return message + return {**message, "content": [_file_part_as_image_url(part) for part in parts]} + + +def _typed_message_with_file_parts_as_image_urls(message: AllMessageValues) -> AllMessageValues: + return cast(AllMessageValues, message_with_file_parts_as_image_urls(message)) # cast-ok: same message, image parts + + +def messages_with_file_parts_as_image_urls(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]: + return tuple(_typed_message_with_file_parts_as_image_urls(message) for message in messages) + + def _image_part_url(part: Mapping[str, object]) -> str | None: """The image URL carried by one content part, whichever of the three dialects wrote it.""" part_type: Final = part.get("type") @@ -2073,6 +2176,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" if block.get("cache_control") is None: diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 927caec3350..484bacdc72c 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -20,6 +20,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( OpenAIRealtimeEvents, @@ -1281,24 +1282,29 @@ class RealTimeStreaming: verbose_logger.exception("Connection closed in backend to client send messages - %s", e) close: Final = backend_close_from(e) self._flush_unbilled_transcription_usage() - if self._backend_refused_session(close): - await self.log_backend_refusal(e) - else: - await self.log_messages() + await self._log_session_end(e if self._backend_refused_session(close) else None) return close except asyncio.CancelledError: self._flush_unbilled_transcription_usage() - await self.log_messages() + await self._log_session_end(None) raise except Exception as e: verbose_logger.exception("Error in backend to client send messages: %s", e) self._flush_unbilled_transcription_usage() - await self.log_messages() + await self._log_session_end(None) return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket") def _backend_refused_session(self, close: BackendClose) -> bool: return close.code != 1000 and not self._backend_sent_frames + async def _log_session_end(self, transport_refusal: Exception | None) -> None: + provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal() + refusal: Final = provider_refusal or transport_refusal + if refusal is None: + await self.log_messages() + return + await self.log_backend_refusal(refusal) + async def log_backend_refusal(self, error: Exception) -> None: if not self.logging_obj: return @@ -1691,7 +1697,13 @@ class RealTimeStreaming: # ``_maybe_send_guardrail_turn_detection_update`` would retry. if msg_type == "session.update" and self.provider_config is None: await self._send_session_gate_once() - sent = await self._send_to_backend(message) + try: + sent = await self._send_to_backend(message) + except RealtimeTranscriptionProtocolError as e: + await self.websocket.send_text(realtime_error_event(str(e), error_type="invalid_request_error")) + if self._uses_deferred_backend_setup(): + raise + continue if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/litellm_core_utils/served_output_texts.py b/litellm/litellm_core_utils/served_output_texts.py index 854785f304b..b19973e06c3 100644 --- a/litellm/litellm_core_utils/served_output_texts.py +++ b/litellm/litellm_core_utils/served_output_texts.py @@ -4,12 +4,13 @@ post-call guardrail rewrote rather than the provider response the proxy assemble from __future__ import annotations from collections.abc import Sequence +from itertools import chain from typing import Final, Literal -from pydantic import BaseModel, Field, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import ModelResponse, ModelResponseStream +from litellm.types.utils import ModelResponse, ModelResponseStream, TextCompletionResponse SERVED_OUTPUT_TEXTS_KEY: Final = "served_output_texts" @@ -44,6 +45,19 @@ class _ChatChoices(LiteLLMBaseModel): choices: list[object] +class _TextCompletionChoice(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + index: int | None = None + text: str | None = None + + +class _TextCompletionChunk(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + choices: tuple[_TextCompletionChoice, ...] + + def _as_json_object(response: object) -> dict[str, object] | None: candidate: Final = response.model_dump() if isinstance(response, BaseModel) else response try: @@ -108,8 +122,13 @@ def served_output_texts(response: object) -> ServedTexts | None: def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: - if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks): - return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))) + chat_chunks: Final = tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)) + if chunks and len(chat_chunks) == len(chunks): + return _indexed_texts_by_choice(tuple(chain.from_iterable(_chat_chunk_texts(chunk) for chunk in chat_chunks))) + if chunks and all(isinstance(chunk, TextCompletionResponse) for chunk in chunks): + return _indexed_texts_by_choice( + tuple(chain.from_iterable(_text_completion_chunk_texts(chunk) for chunk in chunks)) + ) from litellm.llms.anthropic.messages.stream_assembly import ( assemble_anthropic_sse_stream, is_anthropic_sse_stream, @@ -121,19 +140,26 @@ def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: return _chat_texts(assembled) if assembled is not None else None -def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int) -> str | None: - contents: Final = tuple( - content - for chunk in chunks +def _chat_chunk_texts(chunk: ModelResponseStream) -> tuple[tuple[int, str | None], ...]: + return tuple( + (choice.index or 0, content if isinstance(content := choice.delta.content, str) else None) for choice in chunk.choices - if (choice.index or 0) == index and isinstance(content := choice.delta.content, str) ) - return "".join(contents) if contents else None -def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None: - choice_count: Final = max(((choice.index or 0) + 1 for chunk in chunks for choice in chunk.choices), default=0) - texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count)) +def _text_completion_chunk_texts(chunk: object) -> tuple[tuple[int, str | None], ...]: + parsed: Final = _TextCompletionChunk.model_validate(_as_json_object(chunk)) + return tuple((choice.index or 0, choice.text) for choice in parsed.choices) + + +def _joined_choice_text(indexed_texts: Sequence[tuple[int, str | None]], index: int) -> str | None: + texts: Final = tuple(text for text_index, text in indexed_texts if text_index == index and text is not None) + return "".join(texts) if texts else None + + +def _indexed_texts_by_choice(indexed_texts: Sequence[tuple[int, str | None]]) -> ServedTexts | None: + choice_count: Final = max((index + 1 for index, _ in indexed_texts), default=0) + texts: Final = tuple(_joined_choice_text(indexed_texts, index) for index in range(choice_count)) return texts if any(text is not None for text in texts) else None @@ -170,6 +196,8 @@ def _choice_with_text(choice: object, text: str | None) -> object: if choice_obj is None or text is None: return choice message: Final = _as_json_object(choice_obj.get("message")) - if message is None or message.get("content") == text: - return choice - return {**choice_obj, "message": {**message, "content": text}} + if message is not None: + return choice if message.get("content") == text else {**choice_obj, "message": {**message, "content": text}} + if isinstance(choice_obj.get("text"), str): + return {**choice_obj, "text": text} + return choice diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 719f75546b9..8e1791c3c92 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,9 +12,14 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + USAGE_COST_BILLED_PROVIDERS, + chunk_usage, + provider_reported_usage_cost, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1352,7 +1357,7 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS def _logged_provider(logging_obj: Optional["Logging"]) -> object: @@ -1696,10 +1701,17 @@ def stream_chunk_builder( # Merge annotations from ALL chunks — providers may spread # them across multiple streaming chunks or send them only in # the final chunk. - all_annotations: Final[list] = [] + all_annotations: Final[list[object]] = [] for ac in annotation_chunks: all_annotations.extend(ac["choices"][0]["delta"]["annotations"]) - response["choices"][0]["message"]["annotations"] = all_annotations + message_annotations, provider_annotations = partition_annotations(all_annotations) + if message_annotations is not None: + response["choices"][0]["message"]["annotations"] = message_annotations + if provider_annotations: + response.choices[0].message.provider_specific_fields = { + **(response.choices[0].message.provider_specific_fields or {}), + "annotations": list(provider_annotations), + } audio_chunks: Final = [ chunk @@ -1762,7 +1774,10 @@ def stream_chunk_builder( if combined_provider_fields: _choice = response.choices[0] - _choice.message.provider_specific_fields = combined_provider_fields + _choice.message.provider_specific_fields = { + **(_choice.message.provider_specific_fields or {}), + **combined_provider_fields, + } completion_output = get_content_from_model_response(response) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 2fe840dc863..591d77bdb16 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -611,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( - ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") + ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom") ) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index d371aab4581..31b0f06251d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation): message, _ = serialize_http_exception_detail(exc.detail) return tuple(anthropic_sse_error_frames(message)) - def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse": import uuid - from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( - FakeAnthropicMessagesStreamIterator, - ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_response_usage, ) from litellm.types.utils import AnthropicMessagesResponse - block_response: Final = AnthropicMessagesResponse( + return AnthropicMessagesResponse( id=f"msg_{uuid.uuid4()}", type="message", role="assistant", content=[{"type": "text", "text": exc.message}], model=exc.model, stop_reason="end_turn", - usage=blocked_response_usage(getattr(exc, "original_response", None)), + usage=blocked_response_usage(exc.original_response), ) - return list(FakeAnthropicMessagesStreamIterator(response=block_response)) + + def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc))) def _block_continuation_chunks( self, exc: "ModifyResponseException", responses_so_far: Sequence[object] diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 8a82ed98991..e1436473426 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -17,6 +17,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import ( update_request_with_filtered_beta, ) @@ -506,11 +507,20 @@ class AnthropicChatCompletion(BaseLLM): The pair stays mutable because the streaming path rewrites it in place (`data["stream"] = True`) before sending. """ - request_headers, data = update_request_with_filtered_beta( + request_headers, filtered_data = update_request_with_filtered_beta( headers=headers, request_data=request_data, provider=custom_llm_provider, ) + data: Final = dict( + strip_beta_gated_request_body( + filtered_data, + request_headers, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) logging_obj.pre_call( input=messages, api_key=api_key, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..3b7fcdea0e4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -51,6 +51,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, @@ -121,6 +122,54 @@ _DECODED_JSON: Final = TypeAdapter(object) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: + if custom_llm_provider == "anthropic": + return True + model_lower: Final = model.lower() + if custom_llm_provider == "bedrock": + return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) + if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": + return "claude" in model_lower + return False + + +def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool: + return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model) + + +def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None: + if not isinstance(custom, dict) or "defer_loading" in tool_dict: + return None + defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading") + return defer_loading if isinstance(defer_loading, bool) else None + + +def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object: + if not isinstance(tool, dict): + return tool + + if "custom" not in tool: + return tool + + tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool) + + custom: Final = tool_dict["custom"] + without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"} + defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None + if defer_loading is None: + return without_custom if drop_custom else tool_dict + + unhoisted: Final = { + key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading" + } + kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {} + return {**without_custom, "defer_loading": defer_loading, **kept_custom} + + +def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]: + return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) @@ -552,6 +601,20 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]: + return ( + *( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) + if self.is_thinking_display_updates_used(thinking) + else () + ), + *( + (ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,) + if isinstance(thinking, dict) and "block_binding" in thinking + else () + ), + ) + def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool: for message in messages: if message_field(message, "role") != "system": @@ -1132,7 +1195,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): *, custom_llm_provider: str, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, ) -> list[str]: """ @@ -1169,13 +1232,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_mid_conversation_output_config_used: betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else () - ) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else () ) - return list(set(betas).union(thinking_display_betas, tool_change_betas)) + return list(set(betas).union(self.thinking_betas(thinking), tool_change_betas)) @staticmethod def _make_api_key_auth_header( @@ -1237,7 +1297,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: str | None = None, use_bearer_for_custom_base: bool = False, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, wif_minted: bool = False, ) -> dict: @@ -1298,7 +1358,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.update(user_anthropic_beta_headers) all_betas: Final = betas.union( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (), + self.thinking_betas(thinking), (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (), ) @@ -1385,7 +1445,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): auth_token=auth_token, file_id_used=file_id_used, is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")), + thinking=optional_params.get("thinking"), is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages), web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index a230294ad3f..f8bf7232d9c 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drop_params, ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..91fc77773d1 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -11,6 +11,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, @@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + drop_params=drops_unsupported_params(litellm_params.get("drop_params")), ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..e60ffe67c8d 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import split_leading_system_run, ) from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + is_claude_model_target, + merge_anthropic_beta_headers, + tools_without_custom_field, +) from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -372,6 +376,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ Transform request to Anthropic CountTokens format. @@ -391,7 +396,19 @@ class AnthropicCountTokensConfig: **MappingProxyType( { key: value - for key, value in (("system", counted_system), ("tools", tools)) + for key, value in ( + ("system", counted_system), + ( + "tools", + None + if tools is None + else tools_without_custom_field( + tools, + hoist_defer_loading=is_claude_model_target("anthropic", model), + drop_custom=drop_params, + ), + ), + ) if value is not None } ), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..ddda759384f 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -123,6 +123,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_image_source_to_openai_url, + anthropic_inline_document, parse_tool_call_arguments, reasoning_content_from_thinking_blocks, with_prompt_cache_breakpoint, @@ -185,6 +186,8 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAssistantToolCall, + ChatCompletionFileObject, + ChatCompletionFileObjectFile, ChatCompletionImageObject, ChatCompletionImageUrlObject, ChatCompletionRedactedThinkingBlock, @@ -203,7 +206,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ToolMessageContentPart, ) -from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper @@ -362,6 +365,9 @@ class AnthropicAdapter: _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) +_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value}) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -495,7 +501,9 @@ class LiteLLMAnthropicMessagesAdapter: for m in ordered_messages: user_message: ChatCompletionUserMessage | None = None tool_message_list: list[ChatCompletionToolMessage] = [] - new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = [] + new_user_content_list: list[ + ChatCompletionTextObject | ChatCompletionImageObject | ChatCompletionFileObject + ] = [] if m["role"] == "system": system_message = self._translate_midturn_system_message_to_openai(m, model) if system_message is not None: @@ -528,13 +536,8 @@ class LiteLLMAnthropicMessagesAdapter: self._add_prompt_cache_breakpoint_if_present(content, image_obj) ) elif content.get("type") == "document": - # Convert Anthropic document format (PDF, etc.) to OpenAI format - source = content.get("source", {}) - openai_image_url = self._translate_anthropic_image_to_openai(cast(dict, source)) - - if openai_image_url: - image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) - doc_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) + doc_obj = self._translate_anthropic_document_to_openai(content) + if doc_obj is not None: self._add_cache_control_if_applicable(content, doc_obj, model) new_user_content_list.append(doc_obj) elif content.get("type") == "tool_result": @@ -817,6 +820,7 @@ class LiteLLMAnthropicMessagesAdapter: "name", "input_schema", "description", + "custom", "cache_control", "strict", "type", @@ -1127,7 +1131,8 @@ class LiteLLMAnthropicMessagesAdapter: """Translate Anthropic thinking to either thinking or reasoning_effort. A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one - speaks that param. Carrying its adaptive effort tier alongside takes two different params, + speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only + ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params, because the two are not interchangeable at the provider mapping below. Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking`` @@ -1158,7 +1163,9 @@ class LiteLLMAnthropicMessagesAdapter: is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + is_claude_target: Final = ( + self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS output_config: Final = anthropic_message_request.get("output_config") if is_claude_target: @@ -1350,7 +1357,7 @@ class LiteLLMAnthropicMessagesAdapter: text: Final = tool_result_text_part(block) return None if text is None else ChatCompletionTextObject(type="text", text=text) case "image" | "document": - return self._tool_result_image_part(block.get("source")) + return self._source_image_url_part(block.get("source")) case "tool_reference": return ChatCompletionToolReferenceObject( type="tool_reference", tool_name=str(block.get("tool_name") or "") @@ -1358,7 +1365,18 @@ class LiteLLMAnthropicMessagesAdapter: case _: return None - def _tool_result_image_part(self, image_source: object) -> ChatCompletionImageObject | None: + def _translate_anthropic_document_to_openai( + self, document: Mapping[str, object] + ) -> ChatCompletionFileObject | ChatCompletionImageObject | None: + inline_document: Final = anthropic_inline_document(document) + if inline_document is None: + return self._source_image_url_part(document.get("source")) + return ChatCompletionFileObject( + type="file", + file=ChatCompletionFileObjectFile(filename=inline_document.filename, file_data=inline_document.file_data), + ) + + def _source_image_url_part(self, image_source: object) -> ChatCompletionImageObject | None: if not isinstance(image_source, dict): return None openai_image_url = self._translate_anthropic_image_to_openai(image_source) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..c086618e829 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -2,10 +2,22 @@ from collections.abc import AsyncIterator, Mapping, Sequence from typing import ClassVar, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.anthropic_beta_gated_fields import ( + log_beta_gated_strips, + strip_beta_gated_native_request, + strip_beta_gated_system_messages, + unstripped_native_request, +) +from litellm.anthropic_beta_headers_manager import ( + get_unsupported_headers, + has_beta_headers_config, + is_beta_header_supported_by_bundled_config, +) +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -19,7 +31,6 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, - ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, AnthropicMessagesRequest, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -29,13 +40,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, + is_claude_model_target, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, + tools_without_custom_field, ) from .mid_conversation_system import ( as_system_content_blocks, @@ -47,6 +61,8 @@ DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" _CALLER_CREDENTIAL_HEADERS: Final = frozenset({"x-api-key", "authorization"}) +_MESSAGE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + def _carries_caller_credential(headers: Mapping[str, str]) -> bool: """Whether the caller sent their own Anthropic credential, in which case this passthrough @@ -81,6 +97,9 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa return response +_MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True @@ -181,6 +200,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return is_claude_model_target(self._resolved_provider, model) + + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools_without_custom_field( + tools, + hoist_defer_loading=self.hoists_wrapped_defer_loading(model), + drop_custom=drops_unsupported_params(drop_params), + ) + @staticmethod def _filter_billing_headers_from_system(system_param): """ @@ -194,7 +223,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -205,7 +234,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: @@ -215,7 +244,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: return system_param - def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: + def _strip_beta_gated_system_turns( + self, messages: Sequence[Mapping[str, object]], model: str, drop_params: object + ) -> Sequence[Mapping[str, object]]: + if not has_beta_headers_config(self._resolved_provider): + return messages + gated: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) + return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else messages + + def _normalize_system_role_messages( + self, anthropic_messages_request: dict, model: str, drop_params: object + ) -> None: """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, Vertex, and Azure Foundry all enforce identically. @@ -263,7 +303,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self.custom_llm_provider, key="supports_mid_conversation_system", ) - else list(convert_mid_conversation_system_turns(messages[leading_count:])) + else list( + convert_mid_conversation_system_turns( + self._strip_beta_gated_system_turns( + messages[leading_count:], # pyright: ignore[reportUnknownArgumentType] # messages comes from the bare request dict + model=model, + drop_params=drop_params, + ) + ) + ) ) if hoisted or remaining != messages: anthropic_messages_request["messages"] = remaining @@ -322,7 +370,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(headers, optional_params, messages), api_base + return self.finalize_messages_headers(headers, optional_params, messages), api_base async def avalidate_anthropic_messages_environment( self, @@ -362,7 +410,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base + return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]: if auth_header is None: @@ -396,7 +444,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return config_allows_workload_identity(self) - def _finalize_messages_headers( + def finalize_messages_headers( self, headers: dict, # mutable-ok: out-param optional_params: dict, # mutable-ok: out-param @@ -605,6 +653,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if thinking_enabled or effort_enabled: optional_params.pop("temperature", None) + def strips_beta_gated_fields_from_final_headers(self) -> bool: + """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke), + which strip beta-gated fields themselves once that list exists, and on providers with no row in + ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail.""" + return has_beta_headers_config(self._resolved_provider) + def transform_anthropic_messages_request( self, model: str, @@ -700,20 +754,47 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. - _tools: Final = anthropic_messages_optional_request_params.get("tools") or [] + tools: Final = anthropic_messages_optional_request_params.get("tools") + _tools: Final = tools or [] _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools) if not _has_advisor: messages = strip_advisor_blocks_from_messages(messages) + gated: Final = ( + strip_beta_gated_native_request( + headers, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + messages, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare list + drop_params=getattr(litellm_params, "drop_params", None), + ) + if self.strips_beta_gated_fields_from_final_headers() + else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + ) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( - drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + drop_non_string_tool_result_texts( + _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages, + getattr(litellm_params, "drop_params", None), + ) ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, ) - return dict(anthropic_messages_request) + request: Final = ( + {**dict(anthropic_messages_request), "thinking": gated.thinking} + if gated.thinking_stripped + else dict(anthropic_messages_request) # pyright: ignore[reportUnknownVariableType] # built from the base signature's bare dict + ) + if not isinstance(tools, list): + return request + return { + **request, + "tools": self.tools_for_upstream( + OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params + ), + } def transform_anthropic_messages_response( self, @@ -844,17 +925,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if AnthropicModelInfo().is_tool_search_used(tools): beta_values.add(get_tool_search_beta_header(custom_llm_provider)) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) - if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking")) - else () - ) + thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking")) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages) else () ) - all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas) + all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas) dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value forwarded_beta_values: Final = ( # pyright: ignore[reportUnknownVariableType] # derived from the bare beta set all_beta_values - {dangerous_tool_use} diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 99a07a1e9bf..9b22170c9db 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import ( supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} +def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]: + if "prompt_cache_breakpoint" not in block: + return cache_control_as_prompt_cache_breakpoint(block) + wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True) + return cache_control_as_prompt_cache_breakpoint( + with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker) + ) + + def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None: """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" + drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): - return cache_control_as_prompt_cache_breakpoint + return ( + _cache_control_as_wire_safe_prompt_cache_breakpoint + if drops_params + else cache_control_as_prompt_cache_breakpoint + ) if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + if drops_params: return _without_prompt_cache_breakpoint return None diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 1dd07c1739b..94c608f90e6 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -13,6 +13,7 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + anthropic_inline_document, encrypted_reasoning_signature, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, @@ -106,27 +107,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: block: Mapping[str, object], ) -> dict[str, str] | None: # mutable-ok: API message payload """Convert an Anthropic document block to a Responses input_file part.""" + inline_document: Final = anthropic_inline_document(block) + if inline_document is not None: + return { + "type": "input_file", + "filename": inline_document.filename, + "file_data": inline_document.file_data, + } raw_source: Final = block.get("source") if not isinstance(raw_source, Mapping): return None source: Final = cast(Mapping[str, object], raw_source) # cast-ok: untrusted client payload - source_type: Final = source.get("type") - if source_type == "base64": - data: Final = source.get("data") - if not isinstance(data, str) or not data: - return None - raw_media_type: Final = source.get("media_type") - media_type: Final = ( - raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf" - ) - raw_title: Final = block.get("title") - filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf" - return { - "type": "input_file", - "filename": filename, - "file_data": f"data:{media_type};base64,{data}", - } - if source_type == "url": + if source.get("type") == "url": url: Final = source.get("url") if not isinstance(url, str) or not url: return None diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 17c178d169e..27e003ce8b6 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -661,12 +661,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + config: Final = AnthropicMessagesConfig() + body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_optional_request_params=filtered, litellm_params=GenericLiteLLMParams(), - headers={}, + headers=config.finalize_messages_headers({}, dict(filtered), list(messages)), # pyright: ignore[reportUnknownMemberType] # the shared helper takes and returns bare dicts ) return MappingProxyType(_JSON_OBJECT.validate_python(body)) except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 0f8afaae681..4b3f263201a 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -13,6 +13,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, @@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")), ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/azure_ai/anthropic/handler.py b/litellm/llms/azure_ai/anthropic/handler.py index ca4dad5ee3e..1a2003e6262 100644 --- a/litellm/llms/azure_ai/anthropic/handler.py +++ b/litellm/llms/azure_ai/anthropic/handler.py @@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import httpx +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from litellm.llms.anthropic.chat.transformation import JsonModeRequest @@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): litellm_params=litellm_params, ) - data: Final = config.transform_request( + request_body: Final[Mapping[str, object]] = config.transform_request( model=model, messages=messages, optional_params=optional_params, litellm_params=litellm_params, headers=headers, ) + data: Final = dict( + strip_beta_gated_request_body( + request_body, + headers, # pyright: ignore[reportUnknownArgumentType] # validate_environment returns a bare dict + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 6b2da96de23..1f9ded827b6 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -145,6 +145,8 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) return anthropic_messages_request diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index c75213c3e2c..6516827359d 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -107,9 +107,11 @@ class AzureAnthropicConfig(AnthropicConfig): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) # Merge headers - Azure auth (api-key or Authorization) takes precedence - headers = {**anthropic_headers, **headers} + merged_beta_header: Final = {k: v for k, v in anthropic_headers.items() if k == "anthropic-beta"} + headers = {**anthropic_headers, **headers, **merged_beta_header} # Ensure anthropic-version header is set if "anthropic-version" not in headers: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..d382d653a53 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,7 +1,7 @@ import copy import enum import re -from collections.abc import AsyncIterator, Collection, Iterator +from collections.abc import AsyncIterator, Collection, Iterator, Mapping from typing import TYPE_CHECKING, Final, cast import httpx @@ -10,10 +10,11 @@ from httpx import Response import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( - audio_or_image_in_message_content, + content_parts, convert_content_list_to_str, drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, + is_pdf_file_part, ) from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( @@ -46,6 +47,20 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ( "cache_control", ) +LIST_ONLY_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_audio"}) + + +def _is_list_only_content_part(part: object) -> bool: + match part: + case {"type": str() as part_type} if part_type in LIST_ONLY_CONTENT_PART_TYPES: + return True + case _: + return is_pdf_file_part(part) + + +def _has_list_only_content_part(message: Mapping[str, object]) -> bool: + return any(_is_list_only_content_part(part) for part in content_parts(message)) + class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator): def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract @@ -106,7 +121,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( @@ -246,7 +261,7 @@ class AzureAIStudioConfig(OpenAIConfig): these with "Extra inputs are not permitted", which breaks multi-turn Anthropic-format clients that echo thinking blocks back as history. 2. Transforms list content to a string. - 3. If message contains an image or audio, send as is (user-intended) + 3. If message contains an image, audio, or PDF file part, send as is (user-intended) Operates on a deep copy so the caller's messages keep their thinking blocks and provider metadata, which a fallback to another provider still needs. @@ -257,8 +272,7 @@ class AzureAIStudioConfig(OpenAIConfig): for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: filter_value_from_dict(message_dict, field) - # Do nothing if the message contains an image or audio - if audio_or_image_in_message_content(message): + if _has_list_only_content_part(message): continue texts = convert_content_list_to_str(message=message) diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 35b7642d7de..2e5514b3975 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -5,8 +5,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm._lazy_imports import get_default_encoding from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, @@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import ( relayed_body, strip_leading_model_segment, ) +from litellm.llms.cohere.embed.transformation import ( + CohereEmbeddingRowShape, + CohereImageTokenBilling, + cohere_embedding_response, +) from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse from litellm.types.llms.openai import AllMessageValues from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response @@ -35,6 +41,7 @@ if TYPE_CHECKING: EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) +ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True)) def api_version_from(litellm_params: Mapping[str, object]) -> str | None: @@ -85,9 +92,29 @@ def relay_query_params( return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) +def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]: + try: + return ECHOED_TEXTS.validate_python(body.get("texts")) + except ValidationError: + return () + + +def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse: + return cohere_embedding_response( + body, + None, + get_default_encoding(), + echoed_texts(body), + CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS, + CohereEmbeddingRowShape.TYPED, + ) + + FOUNDRY_RELAY_SHAPES: Final = ( RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), + RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response), + RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response), ) diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index da734cf8189..b19915648ab 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -232,7 +232,7 @@ class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: mapped: Final = super().map_openai_params_for_deployment( response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index 1f4ad29fa74..f1539d16b30 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -4,7 +4,6 @@ from types import TracebackType from typing import TYPE_CHECKING, Any, Protocol import httpx -from typing_extensions import Self from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( @@ -24,7 +23,7 @@ else: class RealtimeBackend(Protocol): - async def __aenter__(self) -> Self: ... + async def __aenter__(self) -> object: ... async def __aexit__( self, @@ -97,9 +96,15 @@ class BaseRealtimeConfig(ABC): def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None + def session_refusal(self) -> Exception | None: + return None + async def open_backend(self, url: str, headers: Mapping[str, str]) -> RealtimeBackend | None: return None + def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + return backend + def transform_session_created_event( self, model: str, diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 6885183aeba..4924c5a5acc 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -226,6 +226,9 @@ class BaseVectorStoreConfig: ``get_complete_url`` adds a query string, like Azure, override this.""" return f"{api_base}/{encoded_vector_store_id}" + def get_vector_store_list_url(self, api_base: str) -> str: + return api_base + def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..443e722168d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, @@ -117,6 +119,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1192,11 +1195,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params @@ -1828,7 +1827,16 @@ class AmazonConverseConfig(BaseConfig): filtered_tools.append(tool) base_model: Final = BedrockModelInfo.get_base_model(model) - client_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + header_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + client_beta_list: Final = header_beta_list or filter_and_transform_beta_headers( + sorted( + active_beta_values( + {}, + additional_request_params.get("anthropic_beta"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + ), + provider="bedrock_converse", + ) eager_beta: Final = ( (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,) if base_model.startswith("anthropic") @@ -1836,7 +1844,15 @@ class AmazonConverseConfig(BaseConfig): and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list else () ) - anthropic_beta_list: Final = [*client_beta_list, *eager_beta] + thinking_betas: Final = AnthropicModelInfo().thinking_betas( + additional_request_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + missing_thinking_betas: Final = ( + tuple(beta for beta in thinking_betas if beta not in client_beta_list) + if base_model.startswith("anthropic") + else () + ) + anthropic_beta_list: Final = [*client_beta_list, *eager_beta, *missing_thinking_betas] # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): @@ -1906,7 +1922,8 @@ class AmazonConverseConfig(BaseConfig): else: computer_use_header = "computer-use-2024-10-22" - anthropic_beta_list.append(computer_use_header) + if computer_use_header not in anthropic_beta_list: + anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format transformed_computer_tools: Final = self._transform_computer_use_tools(computer_use_tools) additional_request_params["tools"] = transformed_computer_tools @@ -2031,6 +2048,16 @@ class AmazonConverseConfig(BaseConfig): bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( original_tools, model, headers, additional_request_params, litellm_params, deployment_model ) + requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict + body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict + gated_thinking: Final = strip_beta_gated_thinking( + requested_thinking, # pyright: ignore[reportUnknownArgumentType] # same bare dict + active_beta_values({}, body_betas) | frozenset(anthropic_beta_list), # pyright: ignore[reportUnknownArgumentType] # _process_tools_and_beta hands back a bare list + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse") + additional_request_params["thinking"] = gated_thinking.thinking # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index eb6ed5c75e3..e32df0bbdbb 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import TypeAdapter +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if beta_list: _anthropic_request["anthropic_beta"] = beta_list - return _anthropic_request + return dict( + strip_beta_gated_request_body( + _anthropic_request, # pyright: ignore[reportUnknownArgumentType] # built by the base signature's bare dict transform + {}, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the base signature's bare dict + model=model, + provider="bedrock_invoke", + ) + ) @property def uses_async_transform_request(self) -> bool: @@ -294,7 +303,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) - is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking) client_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) user_beta_set: Final = ( @@ -312,7 +320,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=is_thinking_display_updates_used, + thinking=thinking, ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 0194f1c7e45..a5ecc37d394 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): ), code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")), container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params), + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index fde995c7907..4175479f068 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -15,7 +15,11 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ( + ReadOnly, + TypedDict, + TypeIs, # noqa: TID251 # narrows the caller's tools without copying them +) if TYPE_CHECKING: from botocore.eventstream import EventStreamMessage @@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys( request_body["output_config"] = {"format": preserved_format} # rebind-ok: out-param +_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: JSON tool objects # mutable-ok: fallbacks reuse tools edited in place + return isinstance(value, dict) + + +def _normalize_custom_field_on_tool(tool: object) -> None: + if not _is_str_keyed_dict(tool): + return + custom: Final = tool.pop("custom", None) + if not _is_str_keyed_dict(custom) or "defer_loading" in tool: + return + deferred: Final = custom.get("defer_loading") + if isinstance(deferred, bool): + tool["defer_loading"] = deferred # rebind-ok: a fallback reuses the caller's tool, as on main + + def normalize_custom_field_on_tools(request_body: dict) -> None: """ Drop the ``custom`` field from each tool, first hoisting a boolean @@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None: tools: Final = request_body.get("tools") if not tools or not isinstance(tools, list): return - for tool in tools: - if not isinstance(tool, dict): - continue - custom: dict[str, object] | None = tool.pop("custom", None) - if not isinstance(custom, dict) or "defer_loading" in tool: - continue - deferred: object = custom.get("defer_loading") - if isinstance(deferred, bool): - tool["defer_loading"] = deferred + for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools): + _normalize_custom_field_on_tool(tool) _TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -1314,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 34e622b5e8d..5cd26235ba2 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url @@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None ) endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name) + tools: Final = request.get("tools") body: Final = self._anthropic_config.transform_request_to_count_tokens( model=resolved_model, messages=request["messages"], - tools=request.get("tools"), + tools=None + if tools is None + else _TOOLS.validate_python( + tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True) + ), system=request.get("system"), optional_params=_COUNT_OPTIONS.validate_python( {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES} diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index bf9c119f6f2..df5a3827c1f 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -12,6 +12,7 @@ from typing import Final, Literal from pydantic import JsonValue from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import get_bedrock_base_model @@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM): # For InvokeModel, we need to provide the raw body that would be sent to the model # Remove the 'model' field from the body as it's not part of the model input - body_data: Final = {k: v for k, v in request_data.items() if k != "model"} + tools: Final = request_data.get("tools") + body_data: Final = { + **{k: v for k, v in request_data.items() if k != "model"}, + **( + {"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)} + if isinstance(tools, list) + else {} + ), + } if "messages" in body_data: # Bedrock validates the body against the model's InvokeModel schema; diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 4d27d94664f..2f0ed4d0e82 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,9 +9,11 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx +from pydantic import TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL +from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( @@ -36,7 +38,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError +from ..common_utils import BedrockError, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -64,12 +66,21 @@ def _sign_get_request( return request.prepare() +_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def _without_region_path(model: str) -> str: + region, model_id = split_bedrock_region_path(model) + return model if region is None else model_id + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, + region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -77,15 +88,18 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, + region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, + region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) + explicit_region_name: Final = optional_params.pop("aws_region_name", None) + aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name ### SET REGION NAME ### if aws_region_name is None: @@ -184,14 +198,17 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, is_async_invoke: bool | None = False, batch_data: list[dict] | None = None, + aws_region_name: str | None = None, ) -> EmbeddingResponse | None: """ Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None + bedrock_model: Final = _without_region_path(model) + is_async_job: Final = bool(is_async_invoke) and len(response_list) == 1 and "invocationArn" in response_list[0] # Handle async invoke responses (single response with invocationArn) - if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: + if is_async_job: if provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response( response=response_list[0], model=model @@ -227,15 +244,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -251,6 +268,8 @@ class BedrockEmbedding(BaseAWSLLM): ########################################################## if returned_response is None: raise Exception(f"Unable to map model response to known provider format. model={model}") + if aws_region_name is not None and not is_async_job: + set_hidden_param(returned_response, "region_name", aws_region_name) return returned_response def _single_func_embeddings( @@ -319,6 +338,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) async def _async_single_func_embeddings( @@ -390,6 +410,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) def embeddings( @@ -409,26 +430,29 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: + model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None + has_async_invoke: Final = "async_invoke/" in model + model_without_async_route: Final = model.replace("async_invoke/", "", 1) + path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None + bedrock_model: Final = _without_region_path(model_without_async_route) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key) + optional_params, + bearer_token=bedrock_bearer_token(api_key), + region_from_model=self._get_aws_region_from_model_arn(model_id_param) or path_region, ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed + unencoded_model_id: Final = model_id_param or bedrock_model modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model, + model=bedrock_model, model_id=unencoded_model_id, ) - # Check async invoke needs to be used - has_async_invoke: Final = "async_invoke/" in model - if has_async_invoke: - model = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(model) + provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {model}. " + f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -441,9 +465,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=model, input=input, inference_params=inference_params + model=bedrock_model, input=input, inference_params=inference_params ) - elif provider == "amazon" and model in [ + elif provider == "amazon" and bedrock_model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -451,28 +475,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - model, + bedrock_model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -485,7 +509,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -527,7 +551,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -541,7 +565,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -570,8 +594,9 @@ class BedrockEmbedding(BaseAWSLLM): ## ROUTING ## # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} + set_hidden_param(model_response, "region_name", aws_region_name) return cohere_embedding( - model=model, + model=model_without_async_route, input=input, model_response=model_response, logging_obj=logging_obj, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index e1e15861a90..e5d6a168d57 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -7,6 +7,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, @@ -102,6 +103,9 @@ class AmazonAnthropicClaudeMessagesConfig( def should_strip_billing_metadata(self) -> bool: return True + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools + def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return True @@ -546,6 +550,9 @@ class AmazonAnthropicClaudeMessagesConfig( "edits": retained_edits, } + def strips_beta_gated_fields_from_final_headers(self) -> bool: + return False + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, @@ -581,9 +588,7 @@ class AmazonAnthropicClaudeMessagesConfig( ), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used( - anthropic_messages_request.get("thinking") - ), + thinking=anthropic_messages_request.get("thinking"), is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used( outgoing_messages_typed ), @@ -754,7 +759,9 @@ class AmazonAnthropicClaudeMessagesConfig( ), "display": "updates", } - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### @@ -822,6 +829,20 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict + gated: Final = strip_beta_gated_native_request( + {}, + final_request, + final_request.get("messages"), + body_betas=filtered_betas, + drop_params=getattr(litellm_params, "drop_params", None), + ) + log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke") + if gated.thinking_stripped: + anthropic_messages_request["thinking"] = gated.thinking + if gated.messages_stripped: + anthropic_messages_request["messages"] = list(gated.messages) + outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request) if outbound_tools is not None: anthropic_messages_request["tools"] = outbound_tools diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/cohere/embed/transformation.py b/litellm/llms/cohere/embed/transformation.py index 590164ffa0f..1477e45997b 100644 --- a/litellm/llms/cohere/embed/transformation.py +++ b/litellm/llms/cohere/embed/transformation.py @@ -10,11 +10,13 @@ Convers Docs - https://docs.cohere.com/v2/reference/embed """ -from collections.abc import Iterable, Mapping, Sized -from typing import Final, Protocol, cast +from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized +from enum import Enum +from itertools import chain, starmap +from typing import Final, Protocol, TypeAlias, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter import litellm from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE @@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True)) _DECODED_JSON: Final = TypeAdapter(object) class _SupportsEncode(Protocol): """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" - def encode(self, text: str, /) -> Sized: ... + def encode_ordinary(self, text: str, /) -> Sized: ... + + +class CohereImageTokenBilling(Enum): + SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens" + FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens" + + +class CohereEmbeddingRowShape(Enum): + OPENAI = "openai" + TYPED = "typed" + + +class _CohereBilledUnits(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + input_tokens: int | None = None + image_tokens: int | None = None + images: int | None = None + + +class _CohereEmbedMeta(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits) + + +_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object] + + +class _CohereEmbedBody(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + embeddings: _CohereEmbeddings + meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta) + + +def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors)) + + +def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({**row, "type": embedding_type} for row in _embedding_rows(vectors)) + + +def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]: + if not isinstance(embeddings, Mapping): + return _embedding_rows(embeddings) + match row_shape: + case CohereEmbeddingRowShape.OPENAI: + return chain.from_iterable(map(_embedding_rows, embeddings.values())) + case CohereEmbeddingRowShape.TYPED: + return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items())) + + +def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=prompt_tokens_details, + ) + + +def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage: + match image_billing: + case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS: + image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0) + return _prompt_usage( + (billed_units.input_tokens or 0) + image_units, + PromptTokensDetailsWrapper( + text_tokens=billed_units.input_tokens, + image_tokens=image_units or None, + image_count=billed_units.images, + ), + ) + case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS: + return _prompt_usage(billed_units.input_tokens or 0, None) + + +def _cohere_embed_usage( + billed_units: _CohereBilledUnits, + input_texts: Iterable[str], + encoding: _SupportsEncode, + image_billing: CohereImageTokenBilling, +) -> Usage: + if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None: + return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None) + return _billed_usage(billed_units, image_billing) + + +def cohere_embedding_response( + body: Mapping[str, object], + model: str | None, + encoding: _SupportsEncode, + input_texts: Iterable[str], + image_billing: CohereImageTokenBilling, + row_shape: CohereEmbeddingRowShape, +) -> EmbeddingResponse: + cohere_body: Final = _CohereEmbedBody.model_validate(body) + return EmbeddingResponse( + model=model, + data=list(_response_rows(cohere_body.embeddings, row_shape)), + usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing), + ) class CohereEmbeddingConfig(BaseEmbeddingConfig): @@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): ), ) - def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: - input_tokens = 0 - - text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") - - image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images") - - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if image_tokens is None and text_tokens is None: - for text in input: - input_tokens += len(encoding.encode(text)) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - image_tokens=image_tokens, - text_tokens=text_tokens, - ) - if image_tokens: - input_tokens += image_tokens - if text_tokens: - input_tokens += text_tokens - - return Usage( - prompt_tokens=input_tokens, - completion_tokens=0, - total_tokens=input_tokens, - prompt_tokens_details=prompt_tokens_details, - ) - - def _transform_response( + def transform_native_embed_response( self, response: httpx.Response, api_key: str | None, logging_obj: LiteLLMLoggingObj, - data: dict | CohereEmbeddingRequest, - model_response: EmbeddingResponse, + data: Mapping[str, object] | CohereEmbeddingRequest, model: str, encoding: _SupportsEncode, - input: list, + input: Sequence[str], + row_shape: CohereEmbeddingRowShape, ) -> EmbeddingResponse: response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING @@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): additional_args={"complete_input_dict": data}, original_response=response_json, ) - """ - response - { - 'object': "list", - 'data': [ - - ] - 'model', - 'usage' - } - """ - payload: Final = _JSON_OBJECT.validate_python(response_json) - embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"]) - output_data: Final = [] - for k, embedding_list in embeddings.items(): - for idx, embedding in enumerate(embedding_list): - output_data.append({"object": "embedding", "index": idx, "embedding": embedding}) - model_response.object = "list" - model_response.data = output_data - model_response.model = model - input_tokens = 0 - for text in input: - input_tokens += len(encoding.encode(text)) - - setattr( - model_response, - "usage", - self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))), + return cohere_embedding_response( + _JSON_OBJECT.validate_python(response_json), + model, + encoding, + input, + CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS, + row_shape, ) - return model_response - def transform_embedding_response( self, model: str, @@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, litellm_params: dict, ) -> EmbeddingResponse: - return self._transform_response( + return self.transform_native_embed_response( response=raw_response, api_key=api_key, logging_obj=logging_obj, data=request_data, - model_response=model_response, model=model, encoding=litellm.encoding, input=logging_obj.model_call_details["input"], + row_shape=CohereEmbeddingRowShape.OPENAI, ) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9700465ebc3..c970b507288 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -2577,6 +2581,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **litellm_params.per_second_pricing(), **vertex_location_params, **anthropic_messages_optional_request_params, }, @@ -6275,9 +6280,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) @@ -6920,7 +6923,7 @@ class BaseLLMHTTPHandler: try: ssl_context: Final = realtime_ssl_for_url(url) provider_backend: Final = await provider_config.open_backend(url, headers) - backend_ws: Final = ( + backend_ws: Final = provider_config.wrap_backend( provider_backend if provider_backend is not None else await self._open_realtime_backend_ws(websockets, url, headers, ssl_context) @@ -6958,9 +6961,7 @@ class BaseLLMHTTPHandler: await backend_ws.send(_session_config) realtime_streaming.session_configuration_request = _session_config - # For providers that defer setup until client session.update, optionally - # send synthetic session.created to unblock clients waiting on connect. - if not provider_config.requires_session_configuration(): + if not _session_config: synthetic_session: Final = provider_config.transform_session_created_event( model=model, logging_session_id=logging_obj.litellm_trace_id, @@ -11149,7 +11150,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url: Final = api_base + url: Final = vector_store_provider_config.get_vector_store_list_url(api_base) params: Final[dict[str, object]] = {} if after is not None: @@ -11228,7 +11229,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - url: Final = api_base + url: Final = vector_store_provider_config.get_vector_store_list_url(api_base) params: Final[dict[str, object]] = {} if after is not None: diff --git a/litellm/llms/dashscope/chat/transformation.py b/litellm/llms/dashscope/chat/transformation.py index 966d1a01b8d..dfff9cec298 100644 --- a/litellm/llms/dashscope/chat/transformation.py +++ b/litellm/llms/dashscope/chat/transformation.py @@ -10,6 +10,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from ...openai.chat.gpt_transformation import OpenAIGPTConfig +from ..common_utils import resolve_dashscope_openai_api_base class DashScopeChatConfig(OpenAIGPTConfig): @@ -55,11 +56,9 @@ class DashScopeChatConfig(OpenAIGPTConfig): def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: - api_base = ( - api_base or get_secret_str("DASHSCOPE_API_BASE") or "https://dashscope.aliyuncs.com/compatible-mode/v1" - ) + resolved_api_base: Final = resolve_dashscope_openai_api_base(api_base) dynamic_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY") - return api_base, dynamic_api_key + return resolved_api_base, dynamic_api_key def get_openai_compatible_provider_info( self, @@ -69,7 +68,7 @@ class DashScopeChatConfig(OpenAIGPTConfig): return self._get_openai_compatible_provider_info(api_base, api_key) def _resolve_chat_api_base(self, api_base: str | None) -> str: - return api_base or "https://dashscope.aliyuncs.com/compatible-mode/v1" + return resolve_dashscope_openai_api_base(api_base) def get_complete_url( self, diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 3e2e718a469..5ad2aa17a3a 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -2,6 +2,7 @@ Common utilities for the DashScope LLM provider. """ +from dataclasses import dataclass from typing import TYPE_CHECKING, Final from urllib.parse import urlparse @@ -19,10 +20,123 @@ if TYPE_CHECKING: from litellm.llms.base_llm.videos.transformation import BaseVideoConfig DASHSCOPE_CHAT_COMPATIBLE_PATH: Final = "/compatible-mode/v1" -DASHSCOPE_RERANK_PATH: Final = "/compatible-api/v1/reranks" +DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic" +DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1" +DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1" +DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks" +DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com" + +_STANDARD_DASHSCOPE_HOSTS: Final = frozenset( + { + "dashscope.aliyuncs.com", + "dashscope-intl.aliyuncs.com", + "dashscope-us.aliyuncs.com", + } +) +_WORKSPACE_HOST_SUFFIX: Final = ".maas.aliyuncs.com" +_EXCLUDED_WORKSPACE_PREFIXES: Final = ("trial.", "token-plan.") +_PROTOCOL_PATH_PREFIXES: Final = ( + DASHSCOPE_CHAT_COMPATIBLE_PATH, + DASHSCOPE_ANTHROPIC_PATH, + DASHSCOPE_NATIVE_V1_PATH, + DASHSCOPE_COMPATIBLE_API_PATH, +) -def _rerank_base_for_chat_shaped_base(api_base: str | None) -> str | None: +@dataclass(frozen=True, slots=True) +class DashScopeEndpoints: + origin: str + openai: str + anthropic: str + native: str + + +def _is_dashscope_host(host: str) -> bool: + if host in _STANDARD_DASHSCOPE_HOSTS: + return True + return host.endswith(_WORKSPACE_HOST_SUFFIX) and not host.startswith(_EXCLUDED_WORKSPACE_PREFIXES) + + +def _dashscope_origin(api_base: str | None) -> str | None: + if not api_base: + return None + parsed: Final = urlparse(api_base) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + return None + if parsed.query or parsed.fragment: + return None + host: Final = (parsed.hostname or "").lower() + if not _is_dashscope_host(host): + return None + path: Final = parsed.path.rstrip("/") + if path and not any(path == prefix or path.startswith(f"{prefix}/") for prefix in _PROTOCOL_PATH_PREFIXES): + return None + return f"{parsed.scheme}://{parsed.netloc}" + + +def resolve_dashscope_endpoints(api_base: str | None) -> DashScopeEndpoints | None: + """ + Derive DashScope's protocol-specific bases from any standard DashScope base. + + Accepts the domain root, an OpenAI-compatible base, an Anthropic Messages + base, or a native `/api/v1` base for `dashscope.aliyuncs.com`, + `dashscope-intl.aliyuncs.com`, `dashscope-us.aliyuncs.com`, and + `{workspace}.{region}.maas.aliyuncs.com`. Returns None for any other host + or path so callers keep the caller-supplied base untouched. + """ + origin: Final = _dashscope_origin(api_base) + if origin is None: + return None + return DashScopeEndpoints( + origin=origin, + openai=f"{origin}{DASHSCOPE_CHAT_COMPATIBLE_PATH}", + anthropic=f"{origin}{DASHSCOPE_ANTHROPIC_PATH}", + native=f"{origin}{DASHSCOPE_NATIVE_V1_PATH}", + ) + + +def is_dashscope_native_endpoint_url(api_base: str) -> bool: + return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/") + + +def resolve_dashscope_openai_api_base( + api_base: str | None, + env_var: str = "DASHSCOPE_API_BASE", + default: str = f"{DASHSCOPE_DEFAULT_ORIGIN}{DASHSCOPE_CHAT_COMPATIBLE_PATH}", +) -> str: + candidate: Final = api_base or get_secret_str(env_var) + if candidate and is_dashscope_native_endpoint_url(candidate): + return candidate + endpoints: Final = resolve_dashscope_endpoints(candidate) + if endpoints is not None: + return endpoints.openai + return candidate or default + + +def resolve_dashscope_anthropic_api_base( + api_base: str | None, + env_var: str = "DASHSCOPE_API_BASE", + default: str = f"{DASHSCOPE_DEFAULT_ORIGIN}{DASHSCOPE_ANTHROPIC_PATH}", +) -> str: + candidate: Final = api_base or get_secret_str(env_var) + endpoints: Final = resolve_dashscope_endpoints(candidate) + if endpoints is not None: + return endpoints.anthropic + return candidate or default + + +def dashscope_base_selects_anthropic_messages(api_base: str | None) -> bool: + candidate: Final = api_base or get_secret_str("DASHSCOPE_API_BASE") + if not candidate: + return False + path: Final = urlparse(candidate).path.rstrip("/") + if path.endswith(DASHSCOPE_ANTHROPIC_PATH) or f"{DASHSCOPE_ANTHROPIC_PATH}/" in path: + return True + return not path and resolve_dashscope_endpoints(candidate) is not None + + +def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None: if api_base is None: return None parsed: Final = urlparse(api_base) @@ -30,14 +144,24 @@ def _rerank_base_for_chat_shaped_base(api_base: str | None) -> str | None: on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com") if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: return None - return f"{parsed.scheme}://{parsed.netloc}{DASHSCOPE_RERANK_PATH}" + return f"{parsed.scheme}://{parsed.netloc}" + + +def _is_compatible_api_base(api_base: str) -> bool: + path: Final = urlparse(api_base).path.rstrip("/") + return path == DASHSCOPE_COMPATIBLE_API_PATH or path.startswith(f"{DASHSCOPE_COMPATIBLE_API_PATH}/") def resolve_dashscope_family_rerank_api_base(api_base: str | None, env_var: str, default_rerank_base: str) -> str: - remapped: Final = _rerank_base_for_chat_shaped_base(api_base) - if api_base is not None and remapped is None: + if api_base is None: + return get_secret_str(env_var) or default_rerank_base + if _is_compatible_api_base(api_base): return api_base - return get_secret_str(env_var) or remapped or default_rerank_base + chat_origin: Final = _chat_shaped_aliyun_origin(api_base) + if chat_origin is not None: + return get_secret_str(env_var) or f"{chat_origin}{DASHSCOPE_RERANK_PATH}" + endpoints: Final = resolve_dashscope_endpoints(api_base) + return f"{endpoints.origin}{DASHSCOPE_RERANK_PATH}" if endpoints is not None else api_base def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbeddingConfig": diff --git a/litellm/llms/dashscope/embed/transformation.py b/litellm/llms/dashscope/embed/transformation.py index 35132346261..0f01a4b3172 100644 --- a/litellm/llms/dashscope/embed/transformation.py +++ b/litellm/llms/dashscope/embed/transformation.py @@ -25,7 +25,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage -from ..common_utils import DashScopeError +from ..common_utils import DashScopeError, resolve_dashscope_openai_api_base DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1" @@ -91,7 +91,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): return resolved_api_key def _resolve_embedding_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("DASHSCOPE_API_BASE") or DEFAULT_API_BASE + return resolve_dashscope_openai_api_base(api_base, default=DEFAULT_API_BASE) def validate_environment( self, diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index d1faa6c6cd0..2f528b5a2c3 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -39,6 +39,13 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ImageObject, ImageResponse +from ..common_utils import ( + DASHSCOPE_CHAT_COMPATIBLE_PATH, + DASHSCOPE_IMAGE_PATH, + is_dashscope_native_endpoint_url, + resolve_dashscope_endpoints, +) + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer @@ -53,8 +60,6 @@ _DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" -CHAT_COMPATIBLE_MODE_PATH: Final = "/compatible-mode/v1" - # Maps OpenAI size strings (WxH) to DashScope size strings (W*H) OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { "256x256": "256*256", @@ -102,8 +107,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): raise ValueError("DASHSCOPE_API_KEY is not set") return resolved_api_key - def _resolve_image_api_base(self, image_api_base: str | None) -> str: - return image_api_base or get_secret_str("DASHSCOPE_API_BASE_IMAGE") or DEFAULT_API_BASE + def _image_api_base_env(self) -> str | None: + return get_secret_str("DASHSCOPE_API_BASE_IMAGE") + + def _default_image_api_base(self) -> str: + return DEFAULT_API_BASE + + def _configured_chat_api_base(self) -> str | None: + return get_secret_str("DASHSCOPE_API_BASE") def get_complete_url( self, @@ -114,10 +125,18 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - image_api_base: Final = ( - api_base if api_base and not api_base.rstrip("/").endswith(CHAT_COMPATIBLE_MODE_PATH) else None - ) - return self._resolve_image_api_base(image_api_base) + image_env_api_base: Final = self._image_api_base_env() + configured_api_base: Final = api_base or self._configured_chat_api_base() + if not configured_api_base or configured_api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH): + return image_env_api_base or self._default_image_api_base() + if is_dashscope_native_endpoint_url(configured_api_base): + return configured_api_base + endpoints: Final = resolve_dashscope_endpoints(configured_api_base) + if endpoints is None: + return configured_api_base + if not api_base and image_env_api_base: + return image_env_api_base + return f"{endpoints.origin}{DASHSCOPE_IMAGE_PATH}" def validate_environment( self, diff --git a/litellm/llms/dashscope/messages/__init__.py b/litellm/llms/dashscope/messages/__init__.py new file mode 100644 index 00000000000..08cd184404c --- /dev/null +++ b/litellm/llms/dashscope/messages/__init__.py @@ -0,0 +1,7 @@ +""" +DashScope Anthropic-compatible Messages Module +""" + +from .transformation import DashScopeAnthropicMessagesConfig + +__all__ = ["DashScopeAnthropicMessagesConfig"] diff --git a/litellm/llms/dashscope/messages/transformation.py b/litellm/llms/dashscope/messages/transformation.py new file mode 100644 index 00000000000..9d0d1db91c8 --- /dev/null +++ b/litellm/llms/dashscope/messages/transformation.py @@ -0,0 +1,96 @@ +""" +DashScope Anthropic-compatible Messages transformation config. +""" + +from typing import ClassVar, Final + +import litellm +from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.llms.anthropic.messages.native.transformation import ( + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str + +from ..common_utils import ( + dashscope_base_selects_anthropic_messages, + resolve_dashscope_anthropic_api_base, +) + + +class DashScopeAnthropicMessagesConfig(AnthropicMessagesConfig): + """ + DashScope exposes an Anthropic-compatible Messages API at + ``{origin}/apps/anthropic`` for every DashScope domain, including + ``dashscope.aliyuncs.com`` and ``{workspace}.{region}.maas.aliyuncs.com``. + """ + + _workload_identity_eligible: ClassVar[bool] = False + + @property + def custom_llm_provider(self) -> str | None: + return "dashscope" + + def should_strip_billing_metadata(self) -> bool: + return True + + def handles_web_search_natively(self) -> bool: + return False + + def serves_api_base(self, model: str, api_base: str | None) -> bool: + return dashscope_base_selects_anthropic_messages(api_base) + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return ( + normalize_nonempty_secret_str(api_key) + or normalize_nonempty_secret_str(get_secret_str("DASHSCOPE_API_KEY")) + or normalize_nonempty_secret_str(litellm.api_key) + ) + + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return resolve_dashscope_anthropic_api_base(api_base) + + def validate_anthropic_messages_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors the legacy base override signature + model: str, + messages: list[object], # mutable-ok: mirrors the legacy base override signature + optional_params: dict[str, object], # mutable-ok: mirrors the legacy base override signature + litellm_params: dict[str, object], # mutable-ok: mirrors the legacy base override signature + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: mirrors the base signature + resolved_api_key: Final = self.get_api_key(api_key=api_key) + if resolved_api_key is None: + present_headers: Final = {name.lower() for name in headers} + if "x-api-key" not in present_headers and "authorization" not in present_headers: + raise AuthenticationError( + message=( + "Missing DashScope API key. Set DASHSCOPE_API_KEY or pass api_key explicitly. " + "Anthropic credentials are never sent to a DashScope endpoint." + ), + llm_provider="dashscope", + model=model, + ) + return super().validate_anthropic_messages_environment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportReturnType] # base signature uses an unparameterized dict + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=resolved_api_key, + api_base=api_base, + ) + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict[str, object], # mutable-ok: mirrors the base get_complete_url signature + litellm_params: dict[str, object], # mutable-ok: mirrors the base get_complete_url signature + stream: bool | None = None, + ) -> str: + return ensure_api_route(self.get_api_base(api_base=api_base), "v1/messages") diff --git a/litellm/llms/dashscope/qwen_ai_platform.py b/litellm/llms/dashscope/qwen_ai_platform.py index f7cc50cc82a..db01c8a0a15 100644 --- a/litellm/llms/dashscope/qwen_ai_platform.py +++ b/litellm/llms/dashscope/qwen_ai_platform.py @@ -3,7 +3,7 @@ from typing import Final from litellm.secret_managers.main import get_secret_str from .chat.transformation import DashScopeChatConfig -from .common_utils import resolve_dashscope_family_rerank_api_base +from .common_utils import resolve_dashscope_family_rerank_api_base, resolve_dashscope_openai_api_base from .embed.transformation import DashScopeEmbeddingConfig from .image_generation.transformation import DashScopeImageGenerationConfig from .rerank.transformation import DashScopeRerankConfig @@ -38,7 +38,7 @@ class QwenAIPlatformChatConfig(DashScopeChatConfig): return self._resolve_chat_api_base(api_base), _resolve_qwen_ai_platform_api_key(api_key) def _resolve_chat_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE") or QWEN_AI_PLATFORM_API_BASE + return resolve_dashscope_openai_api_base(api_base, "QWEN_AI_PLATFORM_API_BASE", QWEN_AI_PLATFORM_API_BASE) class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig): @@ -46,7 +46,7 @@ class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig): return _require_qwen_ai_platform_api_key(api_key) def _resolve_embedding_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE") or QWEN_AI_PLATFORM_API_BASE + return resolve_dashscope_openai_api_base(api_base, "QWEN_AI_PLATFORM_API_BASE", QWEN_AI_PLATFORM_API_BASE) class QwenAIPlatformRerankConfig(DashScopeRerankConfig): @@ -63,8 +63,14 @@ class QwenAIPlatformImageGenerationConfig(DashScopeImageGenerationConfig): def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwen_ai_platform_api_key(api_key) - def _resolve_image_api_base(self, image_api_base: str | None) -> str: - return image_api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_IMAGE") or QWEN_AI_PLATFORM_IMAGE_API_BASE + def _image_api_base_env(self) -> str | None: + return get_secret_str("QWEN_AI_PLATFORM_API_BASE_IMAGE") + + def _default_image_api_base(self) -> str: + return QWEN_AI_PLATFORM_IMAGE_API_BASE + + def _configured_chat_api_base(self) -> str | None: + return get_secret_str("QWEN_AI_PLATFORM_API_BASE") class QwenAIPlatformVideoConfig(DashScopeVideoConfig): diff --git a/litellm/llms/dashscope/qwencloud.py b/litellm/llms/dashscope/qwencloud.py index a12e21b0983..6ae42c62e42 100644 --- a/litellm/llms/dashscope/qwencloud.py +++ b/litellm/llms/dashscope/qwencloud.py @@ -3,7 +3,7 @@ from typing import Final from litellm.secret_managers.main import get_secret_str from .chat.transformation import DashScopeChatConfig -from .common_utils import resolve_dashscope_family_rerank_api_base +from .common_utils import resolve_dashscope_family_rerank_api_base, resolve_dashscope_openai_api_base from .embed.transformation import DashScopeEmbeddingConfig from .image_generation.transformation import DashScopeImageGenerationConfig from .rerank.transformation import DashScopeRerankConfig @@ -38,7 +38,7 @@ class QwenCloudChatConfig(DashScopeChatConfig): return self._resolve_chat_api_base(api_base), _resolve_qwencloud_api_key(api_key) def _resolve_chat_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWENCLOUD_API_BASE") or QWENCLOUD_API_BASE + return resolve_dashscope_openai_api_base(api_base, "QWENCLOUD_API_BASE", QWENCLOUD_API_BASE) class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig): @@ -46,7 +46,7 @@ class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig): return _require_qwencloud_api_key(api_key) def _resolve_embedding_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWENCLOUD_API_BASE") or QWENCLOUD_API_BASE + return resolve_dashscope_openai_api_base(api_base, "QWENCLOUD_API_BASE", QWENCLOUD_API_BASE) class QwenCloudRerankConfig(DashScopeRerankConfig): @@ -63,8 +63,14 @@ class QwenCloudImageGenerationConfig(DashScopeImageGenerationConfig): def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwencloud_api_key(api_key) - def _resolve_image_api_base(self, image_api_base: str | None) -> str: - return image_api_base or get_secret_str("QWENCLOUD_API_BASE_IMAGE") or QWENCLOUD_IMAGE_API_BASE + def _image_api_base_env(self) -> str | None: + return get_secret_str("QWENCLOUD_API_BASE_IMAGE") + + def _default_image_api_base(self) -> str: + return QWENCLOUD_IMAGE_API_BASE + + def _configured_chat_api_base(self) -> str | None: + return get_secret_str("QWENCLOUD_API_BASE") class QwenCloudVideoConfig(DashScopeVideoConfig): diff --git a/litellm/llms/dashscope/videos/transformation.py b/litellm/llms/dashscope/videos/transformation.py index 5c738423a5a..59e42e7b926 100644 --- a/litellm/llms/dashscope/videos/transformation.py +++ b/litellm/llms/dashscope/videos/transformation.py @@ -38,6 +38,8 @@ from litellm.types.videos.utils import ( ) from litellm.utils import get_model_info +from ..common_utils import resolve_dashscope_endpoints + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -206,6 +208,9 @@ def _parse_task_response(raw_response: httpx.Response) -> _TaskResponse: def _normalized_api_base(api_base: str) -> str: + endpoints: Final = resolve_dashscope_endpoints(api_base) + if endpoints is not None: + return endpoints.origin trimmed: Final = api_base.rstrip("/") if trimmed.endswith(DASHSCOPE_COMPATIBLE_MODE_PATH): return trimmed[: -len(DASHSCOPE_COMPATIBLE_MODE_PATH)] diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 48118c23ed7..60081c38073 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( handle_invalid_parallel_tool_calls, @@ -514,6 +515,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): self.update_optional_params_with_thinking_tokens( non_default_params=non_default_params, optional_params=mapped_params ) + gated_thinking: Final = strip_beta_gated_thinking( + mapped_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # mapped_params is the base config's bare dict + frozenset(), + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="databricks") + mapped_params["thinking"] = gated_thinking.thinking return mapped_params diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index e22ad85b5f9..90b63d860d3 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) ("content-type" if key.lower() == "content-type" else key): value for key, value in {**auth_headers, **client_anthropic_headers}.items() } - finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers + finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers headers=normalized_headers, optional_params=optional_params, messages=messages, @@ -191,6 +191,8 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + self._normalize_system_role_messages( # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict return anthropic_messages_request diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 65921b628fb..81a186eadf4 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -38,6 +38,15 @@ else: _EMPTY_FUNCTION: Final[Mapping[str, object]] = MappingProxyType({}) _JSON_VALUE: Final = TypeAdapter(object) +_CONTENT_PART: Final = TypeAdapter(Mapping[str, object]) + + +def _file_part_data(part: Mapping[str, object]) -> str | None: + match part: + case {"type": "file", "file": {"file_data": str() as file_data}} if file_data: + return file_data + case _: + return None def is_valid_json(value: str) -> bool: @@ -295,7 +304,7 @@ class GigaChatConfig(BaseConfig): def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[object]]: """ - Extract text and image attachments from a multimodal message content list. + Extract text, image and file attachments from a multimodal message content list. Args: content: List of content parts (OpenAI multimodal format) @@ -320,6 +329,10 @@ class GigaChatConfig(BaseConfig): file_id = self._upload_image(upload_url) if file_id: attachments.append(file_id) + elif (file_data := _file_part_data(_CONTENT_PART.validate_python(part))) and ( + uploaded_file_id := self._upload_image(file_data) + ): + attachments.append(uploaded_file_id) text: Final = "\n".join(texts) if texts else "" return text, attachments diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 59e1804f453..ee0e7af7e9b 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -1,11 +1,12 @@ import json import os import time +from dataclasses import dataclass from datetime import datetime -from typing import Any, Final +from typing import Any, Final, TypeVar import httpx -from pydantic import ConfigDict, TypeAdapter, with_config +from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -13,7 +14,6 @@ from litellm.litellm_core_utils.asyncify import can_block_current_thread from litellm.llms.custom_httpx.http_handler import get_httpx_client from .common_utils import ( - APIKeyExpiredError, GetAccessTokenError, GetAPIKeyError, GetDeviceCodeError, @@ -27,6 +27,44 @@ DEFAULT_GITHUB_ACCESS_TOKEN_URL: Final = "https://github.com/login/oauth/access_ DEFAULT_GITHUB_API_KEY_URL: Final = "https://api.github.com/copilot_internal/v2/token" +class CopilotEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + api: str | None = None + + +class CopilotCachedKey(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + token: str + expires_at: float + endpoints: CopilotEndpoints | None = None + + +class CopilotCachedEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + endpoints: CopilotEndpoints | None = None + + +ApiKeyFileView = TypeVar("ApiKeyFileView", CopilotCachedKey, CopilotCachedEndpoints) + + +@dataclass(frozen=True, slots=True) +class _UnsavedApiKey: + access_token: str + key: CopilotCachedKey + + +_UNSAVED_API_KEYS: Final[dict[str, _UnsavedApiKey]] = {} # mutable-ok: refreshed keys api-key.json could not hold + + +def _read_token_file(path: str) -> str: + with open(path, "rb") as f: + raw: Final = f.read() + return raw.decode(json.detect_encoding(raw)) + + @with_config(ConfigDict(extra="allow", strict=True)) class _DeviceCode(TypedDict): device_code: ReadOnly[str] @@ -87,12 +125,13 @@ class Authenticator: GetAccessTokenError: If unable to obtain an access token after retries. """ try: - with open(self.access_token_file, "r") as f: - access_token = f.read().strip() - if access_token: - return access_token + saved_token: Final = _read_token_file(self.access_token_file).strip() + if saved_token: + return saved_token except OSError: verbose_logger.warning("No existing access token found or error reading file") + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a text access token: %s", self.access_token_file, e) if not can_block_current_thread(): raise GetAccessTokenError( @@ -135,64 +174,83 @@ class Authenticator: Raises: GetAPIKeyError: If unable to obtain an API key. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info = json.load(f) - if api_key_info.get("expires_at", 0) > datetime.now().timestamp(): - return api_key_info.get("token") - else: - verbose_logger.warning("API key expired, refreshing") - raise APIKeyExpiredError( - message="API key expired", - status_code=401, - ) - except OSError: - verbose_logger.warning("No API key file found or error opening file") - except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API key from file: %s", e) - except APIKeyExpiredError: - pass # Already logged in the try block + unsaved: Final = self._unsaved_api_key() + if unsaved is not None: + return unsaved.token + cached: Final = self._read_api_key_file(CopilotCachedKey) + if cached is not None and cached.expires_at > datetime.now().timestamp(): + return cached.token + if cached is not None: + verbose_logger.warning("API key expired, refreshing") try: - api_key_info = self._refresh_api_key() - with open(self.api_key_file, "w") as f: - json.dump(api_key_info, f) - token: Final = api_key_info.get("token") - if token: - return token - else: - raise GetAPIKeyError( - message="API key response missing token", - status_code=401, - ) - except OSError as e: - verbose_logger.error("Error saving API key to file: %s", e) - raise GetAPIKeyError( - message=f"Failed to save API key: {e}", - status_code=500, - ) + api_key_info: Final = self._refresh_api_key() except RefreshAPIKeyError as e: raise GetAPIKeyError( message=f"Failed to refresh API key: {e}", status_code=401, + ) from e + serialized: Final = json.dumps(api_key_info) + try: + with open(self.api_key_file, "w") as f: + f.write(serialized) + except OSError as e: + verbose_logger.warning( + "Keeping the refreshed API key in memory, it could not be saved to %s: %s", self.api_key_file, e ) + self._keep_unsaved_api_key(serialized) + token: Final = api_key_info.get("token") + if not token: + raise GetAPIKeyError( + message="API key response missing token", + status_code=401, + ) + return token def get_api_base(self) -> str | None: """ - Get the API endpoint from the api-key.json file. + Get the API endpoint of the current API key. Returns: Optional[str]: The GitHub Copilot API endpoint, or None if not found. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info: Final = json.load(f) - endpoints: Final = api_key_info.get("endpoints", {}) - api_endpoint: Final = endpoints.get("api") - return api_endpoint - except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API endpoint from file: %s", e) + cached: Final = self._unsaved_api_key() or self._read_api_key_file(CopilotCachedEndpoints) + if cached is None or cached.endpoints is None: return None + return cached.endpoints.api + + def _read_api_key_file(self, view: type[ApiKeyFileView]) -> ApiKeyFileView | None: + try: + return view.model_validate_json(_read_token_file(self.api_key_file)) + except OSError: + verbose_logger.warning("No API key file found or error opening file") + return None + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a Copilot API key file: %s", self.api_key_file, e) + return None + + def _stored_access_token(self) -> str | None: + try: + access_token: Final = _read_token_file(self.access_token_file).strip() + except (OSError, ValueError): + return None + return access_token or None + + def _unsaved_api_key(self) -> CopilotCachedKey | None: + unsaved: Final = _UNSAVED_API_KEYS.get(self.api_key_file) + if unsaved is None or unsaved.key.expires_at <= time.time(): + return None + return unsaved.key if unsaved.access_token == self._stored_access_token() else None + + def _keep_unsaved_api_key(self, serialized: str) -> None: + access_token: Final = self._stored_access_token() + if access_token is None: + return + try: + key: Final = CopilotCachedKey.model_validate_json(serialized) + except ValueError: + return + _UNSAVED_API_KEYS[self.api_key_file] = _UnsavedApiKey(access_token=access_token, key=key) def _refresh_api_key(self) -> dict[str, Any]: """ diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index e8d37f857e6..8257a75b4a5 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -20,6 +20,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -681,7 +682,9 @@ class OCIChatConfig(BaseConfig): servingMode=serving_mode, chatRequest=OCIChatRequestPayload( apiFormat=vendor.value, - messages=adapt_messages_to_generic_oci_standard(messages), + messages=adapt_messages_to_generic_oci_standard( + list(messages_with_file_parts_as_image_urls(messages)) + ), **self._get_optional_params(vendor, optional_params, model), ), ) diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 0311f6c6590..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -5,8 +5,10 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), + ), + "reasoning_effort", + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 216725b9449..a9d2dcdffb8 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -104,6 +104,19 @@ def is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) +_OPENAI_AGENTS_API_RESOURCES: Final = frozenset({"agents", "vaults"}) + + +def openai_api_resource(path: str) -> str | None: + segments: Final = tuple(segment for segment in path.split("/") if segment) + resource_segments: Final = segments[1:] if segments[:1] == ("v1",) else segments + return next(iter(resource_segments), None) + + +def is_openai_agents_api_path(path: str) -> bool: + return openai_api_resource(path) in _OPENAI_AGENTS_API_RESOURCES + + class OpenAIError(BaseLLMException): def __init__( self, diff --git a/litellm/llms/openai/completion/guardrail_translation/handler.py b/litellm/llms/openai/completion/guardrail_translation/handler.py index c4bc04d4627..27c82cfc004 100644 --- a/litellm/llms/openai/completion/guardrail_translation/handler.py +++ b/litellm/llms/openai/completion/guardrail_translation/handler.py @@ -5,18 +5,30 @@ This module provides guardrail translation support for OpenAI's text completion The handler processes the 'prompt' parameter for guardrails. """ -from collections.abc import Mapping +import json +from collections.abc import Mapping, Sequence +from itertools import chain from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation -from litellm.types.utils import GenericGuardrailAPIInputs +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, + StreamTransformSink, +) +from litellm.llms.base_llm.guardrail_translation.utils import ( + blocked_chat_stream_usage, + stream_item_field, + stream_item_items, +) +from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices, TextCompletionResponse, Usage if TYPE_CHECKING: - from litellm.integrations.custom_guardrail import CustomGuardrail + from fastapi import HTTPException + + from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth - from litellm.types.utils import TextCompletionResponse class OpenAITextCompletionHandler(BaseTranslation): @@ -32,6 +44,9 @@ class OpenAITextCompletionHandler(BaseTranslation): - A list of strings (for batch completions) """ + delivers_ended_stream_rewrites = True + assembles_streamed_response = True + async def process_input_messages( self, data: dict, @@ -125,7 +140,7 @@ class OpenAITextCompletionHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, ) -> "TextCompletionResponse": """ Process output response by applying guardrails to completion text. @@ -162,11 +177,7 @@ class OpenAITextCompletionHandler(BaseTranslation): if "response" not in request_data: request_data["response"] = response - # Add user API key metadata with prefixed keys - if "litellm_metadata" not in request_data: - user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict) - if user_metadata: - request_data["litellm_metadata"] = user_metadata + self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) # Include model information from the response if available @@ -195,3 +206,168 @@ class OpenAITextCompletionHandler(BaseTranslation): ) return response + + async def process_output_streaming_response( + self, + responses_so_far: list[object], # mutable-ok: rewrites the caller's buffered chunks in place + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, + request_data: dict[str, object] | None = None, # mutable-ok: the hooks share and mutate the request payload + stream_transform_sink: StreamTransformSink | None = None, + deliver_ended_stream_rewrites: bool = False, + ) -> Sequence[object]: + chunks: Final = _text_completion_chunks(responses_so_far) + if not self._check_streaming_has_ended(chunks): + return responses_so_far + assembled: Final = self._assemble_ended_stream(chunks) + pre_guardrail_texts: Final = _choice_texts(assembled) + await self.process_output_response( + response=assembled, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=request_data, + ) + if not deliver_ended_stream_rewrites: + return responses_so_far + rewrites: Final = tuple( + after for before, after in zip(pre_guardrail_texts, _choice_texts(assembled)) if after != before + ) + for choice_index, text in rewrites: + self._write_choice_text(chunks, choice_index, text) + return responses_so_far + + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + chunks: Final = _text_completion_chunks(responses_so_far) + if not self._check_streaming_has_ended(chunks): + return None + assembled: Final = self._assemble_ended_stream(chunks) + return StreamingScanKey(texts=tuple(text for _, text in _choice_texts(assembled)), stream_ended=True) + + def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]: + chunks: Final = _text_completion_chunks(responses_so_far) + streamed_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks)) + finished_indices: Final = frozenset( + _choice_index(choice) for choice in streamed_choices if choice.finish_reason + ) + unfinished_indices: Final = tuple( + dict.fromkeys( + _choice_index(choice) for choice in streamed_choices if _choice_index(choice) not in finished_indices + ) + ) + if not unfinished_indices: + return tuple(responses_so_far) + terminator: Final = TextCompletionResponse( + id=chunks[0].id, + model=chunks[0].model, + object="text_completion", + choices=[TextChoices(index=index, text="", finish_reason="length") for index in unfinished_indices], + ) + return (*responses_so_far, terminator) + + def build_block_sse_chunks( + self, + exc: "ModifyResponseException", + stream_started: bool = False, + responses_so_far: Sequence[object] | None = None, + ) -> Sequence[bytes]: + streamed: Final = _text_completion_chunks(responses_so_far or ()) + source: Final = streamed[0] if streamed else TextCompletionResponse(model=exc.model) + message_chunk: Final = TextCompletionResponse( + id=source.id, + created=source.created, + model=source.model or exc.model, + object="text_completion", + choices=[TextChoices(index=0, text=exc.message)], + ).model_copy(update={"usage": None}) + final_chunk: Final = TextCompletionResponse( + id=source.id, + created=source.created, + model=source.model or exc.model, + object="text_completion", + choices=[TextChoices(index=0, text="", finish_reason="content_filter")], + ).model_copy(update={"usage": _blocked_stream_usage(exc.original_response)}) + return tuple( + f"data: {chunk.model_dump_json(exclude_none=True)}\n\n".encode() for chunk in (message_chunk, final_chunk) + ) + + def build_stream_error_items( + self, + exc: "HTTPException", + responses_so_far: Sequence[object] | None = None, + ) -> Sequence[bytes] | None: + from litellm.proxy.common_request_processing import sse_error_payload + + _, error_obj = sse_error_payload(exc) + return (f'data: {{"error": {json.dumps(error_obj)}}}\n\n'.encode(),) + + def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: + streamed_choices: Final = chain.from_iterable(stream_item_items(item, "choices") for item in responses_so_far) + return any(stream_item_field(choice, "finish_reason") is not None for choice in streamed_choices) + + @staticmethod + def _assemble_ended_stream(responses_so_far: Sequence[TextCompletionResponse]) -> TextCompletionResponse: + streamed_choices: Final = tuple(chain.from_iterable(item.choices for item in responses_so_far)) + choice_indices: Final = tuple(dict.fromkeys(_choice_index(choice) for choice in streamed_choices)) + assembled_choices: Final = [ + TextChoices( + index=index, + text="".join(_choice_text(choice) for choice in streamed_choices if _choice_index(choice) == index), + finish_reason=next( + ( + choice.finish_reason + for choice in streamed_choices + if _choice_index(choice) == index and choice.finish_reason + ), + None, + ), + ) + for index in choice_indices + ] + first: Final = responses_so_far[0] + return TextCompletionResponse( + id=first.id, + created=first.created, + model=first.model, + object="text_completion", + choices=assembled_choices, + usage=next((item.usage for item in reversed(responses_so_far) if item.usage is not None), None), + ) + + @staticmethod + def _write_choice_text(responses_so_far: Sequence[TextCompletionResponse], choice_index: int, text: str) -> None: + streamed_choices: Final = chain.from_iterable(item.choices for item in responses_so_far) + head, *tail = (choice for choice in streamed_choices if _choice_index(choice) == choice_index) + head.text = text + for choice in tail: + if isinstance(stream_item_field(choice, "text"), str): + choice.text = "" + + +def _blocked_stream_usage(original_response: object) -> Usage | None: + prompt_tokens, completion_tokens = blocked_chat_stream_usage(original_response) + if isinstance(original_response, list) and prompt_tokens == completion_tokens == 0: + return None + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + + +def _text_completion_chunks(responses_so_far: Sequence[object]) -> tuple[TextCompletionResponse, ...]: + return tuple(item for item in responses_so_far if isinstance(item, TextCompletionResponse)) + + +def _choice_text(choice: TextChoices) -> str: + text: Final = stream_item_field(choice, "text") + return text if isinstance(text, str) else "" + + +def _choice_index(choice: TextChoices) -> int: + return choice.index or 0 + + +def _choice_texts(response: TextCompletionResponse) -> tuple[tuple[int, str], ...]: + return tuple((_choice_index(choice), _choice_text(choice)) for choice in response.choices) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5a3ab8aebea..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -50,7 +50,11 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -206,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index be40220f94d..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + openai_prompt_cache_breakpoint_support, + targets_openai_api, +) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, + prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -27,6 +32,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_a from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -50,6 +56,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -66,6 +75,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"}) _COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) _COMPACT_REJECTED_PARAMS: Final = frozenset( { @@ -93,6 +103,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) +class _DeclaredModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + base_model: object = None + model_info: object = None + + class _DeploymentHost(BaseModel): """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`.""" @@ -216,6 +233,63 @@ def _with_breakpoint_on_last_reading_part( return (*parts[:last], marked_last, *parts[last + 1 :]) +def _json_object_or_none(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None + + +def _json_array_or_empty(value: object) -> Sequence[object]: + return value if isinstance(value, (list, tuple)) else () + + +def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(item) + if fields is None or not _carries_prompt_cache_breakpoint(fields): + return item + wire_safe_fields: Final = { + key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers) + for key, value in fields.items() + if key != "prompt_cache_breakpoint" + } + unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects( + wire_safe_fields.values(), fields.values() + ) + return item if unchanged else wire_safe_fields + + +def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool: + return all(new is old for new, old in zip(rebuilt, originals, strict=True)) + + +def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: + return "prompt_cache_breakpoint" in fields or any( + _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS + ) + + +def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool: + return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts)) + + +def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object: + if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value): + return value + parts: Final = _json_array_or_empty(value) + wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts) + return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts) + + +def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(part) + if fields is None or "prompt_cache_breakpoint" not in fields: + return part + marker: Final = fields["prompt_cache_breakpoint"] + wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None + if keeps_markers and wire_marker == marker: + return part + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, wire_marker) + + class _ReasoningSupportEntry(LiteLLMBaseModel): litellm_provider: str | None = None supports_reasoning: bool | None = None @@ -347,12 +421,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -394,17 +475,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -424,7 +508,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -442,7 +528,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -450,6 +536,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: @@ -544,12 +639,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): validated_input: Final = self._validate_input_param( self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) ) - marked_input: Final = ( - self._input_with_prompt_cache_breakpoints(validated_input) - if self._reads_prompt_cache_breakpoints( - model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options - ) - else validated_input + reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options + ) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints ) maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any( is_anthropic_tool_search_chat_tool(tool) for tool in tools or () @@ -559,20 +653,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if maps_anthropic_tool_search else tools ) - replayable_input: Final = ( - input_without_unanswered_anthropic_tool_search_calls(marked_input) - if maps_anthropic_tool_search - else marked_input - ) stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( - model=model, input=replayable_input, tools=mapped_tools + model=model, + input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), + tools=mapped_tools, ) + if marker_safe_input is not validated_input: + self._strip_caller_input_as_if_unmarked( + model=model, + caller_input=validated_input, + reads_breakpoints=reads_breakpoints, + maps_anthropic_tool_search=maps_anthropic_tool_search, + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + def _replayable_input( + self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool + ) -> str | ResponseInputParam: + marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input + if not maps_anthropic_tool_search: + return marked_input + return input_without_unanswered_anthropic_tool_search_calls(marked_input) + + def _strip_caller_input_as_if_unmarked( + self, + model: str, + caller_input: str | ResponseInputParam, + reads_breakpoints: bool, + maps_anthropic_tool_search: bool, + ) -> None: + self.remove_cache_control_flag_from_input_and_tools( + model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) + ) + + def _input_with_wire_safe_prompt_cache_breakpoints( + self, + model: str, + input: str | ResponseInputParam, + litellm_params: GenericLiteLLMParams, + reads_breakpoints: bool, + ) -> str | ResponseInputParam: + if not isinstance(input, list) or not drops_params(litellm_params): + return input + keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params + ) + wire_safe_items: Final = tuple( + _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input + ) + if _same_objects(wire_safe_items, input): + return input + wire_safe_input: Final = list(wire_safe_items) + return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker + + def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + provider: Final = _provider_name(self.custom_llm_provider) + base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + known_support: Final = ( + openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model)) + ) + return next((support for support in known_support if support is not None), None) is not False + def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url) @@ -688,14 +833,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _without_foreign_tool_call_item_id(item: object) -> object: - if not isinstance(item, dict): + fields: Final = _json_object_or_none(item) + if fields is None: return item - item_type: Final = item.get("type") - item_id: Final = item.get("id") + item_type: Final = fields.get("type") + item_id: Final = fields.get("id") genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix): return item - return {key: value for key, value in item.items() if key != "id"} + return {key: value for key, value in fields.items() if key != "id"} def _sanitized_tool_schemas_for_openai( self, @@ -739,6 +885,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None return base_model if isinstance(base_model, str) and base_model else model + @staticmethod + def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + declared: Final = _DeclaredModel.model_validate(litellm_params) + model_info: Final = _json_object_or_none(declared.model_info) + candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model")) + return next((name for name in candidates if isinstance(name, str) and name), model) + @staticmethod def _sanitized_tool_entry( entry: Mapping[str, object], diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 8daae2a8e6a..a5d825a39d7 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def is_native_messages_route(self) -> bool: return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return False + def validate_anthropic_messages_environment( self, headers: dict[str, str], diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe88ac03e04..ad47b334ed4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -270,6 +270,13 @@ "/v1/responses" ] }, + "kyma": { + "base_url": "https://kymaapi.com/v1", + "api_key_env": "KYMA_API_KEY", + "provider_info": { + "supports_function_calling": true + } + }, "streamlake": { "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", "api_key_env": "STREAMLAKE_API_KEY", diff --git a/litellm/llms/opensearch/__init__.py b/litellm/llms/opensearch/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/opensearch/vector_stores/__init__.py b/litellm/llms/opensearch/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/opensearch/vector_stores/transformation.py b/litellm/llms/opensearch/vector_stores/transformation.py new file mode 100644 index 00000000000..2d862bc8ea6 --- /dev/null +++ b/litellm/llms/opensearch/vector_stores/transformation.py @@ -0,0 +1,415 @@ +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from functools import partial +from typing import TYPE_CHECKING, Final, Literal, NoReturn +from urllib.parse import quote, urlsplit + +import httpx +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.exceptions import AuthenticationError, BadRequestError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.optional_imports import ensure_optional_import +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.vector_store.transformation import ( + BaseQueryEmbeddingVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS, sign_aws_json_post +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.bedrock import AwsAuthParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import EmbeddingResponse +from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VectorStoreCreateOptionalRequestParams, + VectorStoreIndexEndpoints, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from botocore.credentials import Credentials + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DEFAULT_EMBEDDING_FIELD: Final = "embedding" +DEFAULT_TEXT_FIELD: Final = "text" +DEFAULT_MAX_NUM_RESULTS: Final = 10 +MAX_MAX_NUM_RESULTS: Final = 50 +_PARAM_PREFIX: Final = "opensearch_" +_AWS_HOSTS: Final = ( + re.compile(r"\.(?P[a-z0-9-]+)\.(?Paoss|aoss-fips|es)\.amazonaws\.com(?:\.cn)?\Z"), + re.compile(r"\.(?Paoss|aos)\.(?P[a-z0-9-]+)\.on\.aws\Z"), +) +_ACCOUNT_ID_LABEL: Final = re.compile(r"\A\d{12}\.") +_QUERY_VECTOR: Final = TypeAdapter(tuple[float, ...]) +_ATTRIBUTE_VALUE_TYPES: Final = (str, int, float, bool) +_SEARCH_ONLY_MESSAGE: Final = ( + "The OpenSearch vector store is search-only. Create the k-NN index in OpenSearch, " + "then register it here with its name as opensearch_index_name or vector_store_id." +) +_PER_ACCOUNT_MESSAGE: Final = ( + "api_base is an OpenSearch Serverless per-account endpoint, which needs a collection header the vector store " + "does not send. Use the collection endpoint instead, for example https://..aoss.amazonaws.com." +) +_SIGV4_ONLY_MESSAGE: Final = ( + "The OpenSearch vector store signs every request with AWS SigV4 and does not support api_key. " + "Remove api_key and give the vector store AWS credentials instead." +) +_CREDENTIALS_HINT: Final = ( + "Set aws_access_key_id and aws_secret_access_key, aws_role_name, or aws_profile_name on the vector store, " + "or run LiteLLM with an AWS identity." +) + + +def _bad_request(message: str) -> BadRequestError: + return BadRequestError(message=message, model=None, llm_provider="opensearch") + + +def _unauthenticated(reason: str) -> AuthenticationError: + return AuthenticationError( + message=f"No usable AWS credentials to sign the OpenSearch request ({reason}). {_CREDENTIALS_HINT}", + llm_provider="opensearch", + model=None, + ) + + +def _usable_credentials(credentials: object) -> "Credentials": + from botocore.credentials import Credentials + + if isinstance(credentials, Credentials): + return credentials + raise _unauthenticated("none were found") + + +class _OpenSearchParams(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_embedding_model: str | None = None + opensearch_embedding_field: str | None = None + opensearch_index_name: str | None = None + opensearch_text_field: str | None = None + + @property + def embedding_field(self) -> str: + return self.opensearch_embedding_field or DEFAULT_EMBEDDING_FIELD + + @property + def text_field(self) -> str: + return self.opensearch_text_field or DEFAULT_TEXT_FIELD + + def index_name(self, vector_store_id: str) -> str: + return self.opensearch_index_name or vector_store_id + + def require_embedding_model(self) -> str: + if not self.litellm_embedding_model: + raise _bad_request( + "litellm_embedding_model is required for the OpenSearch vector store. It must be the model that " + f"produced the vectors in '{self.embedding_field}', for example openai/text-embedding-3-small." + ) + return self.litellm_embedding_model + + +class _SigningRegion(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + aws_region_name: str | None = None + + +class _SearchOptions(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + max_num_results: int | None = None + filters: Mapping[str, object] | None = None + ranking_options: Mapping[str, object] | None = None + rewrite_query: bool | None = None + + @property + def unsupported(self) -> tuple[str, ...]: + requested: Final = ( + ("filters", self.filters), + ("ranking_options", self.ranking_options), + ("rewrite_query", self.rewrite_query), + ) + return tuple(name for name, value in requested if value) + + @property + def k(self) -> int: + if self.max_num_results is None: + return DEFAULT_MAX_NUM_RESULTS + if not 1 <= self.max_num_results <= MAX_MAX_NUM_RESULTS: + raise _bad_request( + f"max_num_results must be between 1 and {MAX_MAX_NUM_RESULTS}, got {self.max_num_results}" + ) + return self.max_num_results + + +_KNOWN_PARAMS: Final = frozenset(name for name in _OpenSearchParams.model_fields if name.startswith(_PARAM_PREFIX)) + + +class _LoggedSearch(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + query: str | tuple[str, ...] = "" + litellm_params: _OpenSearchParams = _OpenSearchParams() + + @property + def search_query(self) -> str: + return BaseQueryEmbeddingVectorStoreConfig.query_text(self.query) + + +class _Hit(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + id: str = Field(alias="_id") + score: float | None = Field(default=None, alias="_score") + source: Mapping[str, object] = Field(default_factory=dict, alias="_source") + + +class _Hits(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + hits: Sequence[_Hit] + + +class _SearchResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + hits: _Hits + + +def aws_signing_target(url: str) -> tuple[Literal["aoss", "es"], str | None]: + hostname: Final = urlsplit(url).hostname or "" + match: Final = next(filter(None, (pattern.search(hostname) for pattern in _AWS_HOSTS)), None) + if match is None: + return "es", None + return ("aoss" if match["service"].startswith("aoss") else "es"), match["region"] + + +def _search_result(hit: _Hit, params: _OpenSearchParams) -> VectorStoreSearchResult: + text: Final = hit.source.get(params.text_field) + filename: Final = hit.source.get("filename") + return VectorStoreSearchResult( + score=hit.score, + content=[VectorStoreResultContent(type="text", text=text)] if isinstance(text, str) else [], + file_id=hit.id, + filename=filename if isinstance(filename, str) else None, + attributes={ + key: value + for key, value in hit.source.items() + if key not in (params.text_field, params.embedding_field) and isinstance(value, _ATTRIBUTE_VALUE_TYPES) + }, + ) + + +class OpenSearchVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, SignsRequestsWithAWS): + def __init__(self, aws: BaseAWSLLM | None = None) -> None: + self._aws: Final = aws if aws is not None else BaseAWSLLM() + + def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> BaseVectorStoreAuthCredentials: + return BaseVectorStoreAuthCredentials() + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + return VectorStoreIndexEndpoints(read=[], write=[]) + + def validate_environment( + self, headers: Mapping[str, object], litellm_params: GenericLiteLLMParams | None + ) -> dict[str, object]: # mutable-ok: the shared HTTP handler requires writable headers + params: Final = dict(litellm_params or {}) + self._signing_credentials(params, self._signing_region(params, str(params.get("api_base") or ""))) + return {**headers, "Content-Type": "application/json"} + + def _signing_region(self, params: Mapping[str, object], api_base: str) -> str: + host_region: Final = aws_signing_target(api_base)[1] + return self._aws.get_aws_region_name_for_non_llm_api_calls( + host_region or _SigningRegion.model_validate(params).aws_region_name + ) + + def _signing_credentials(self, params: Mapping[str, object], region: str) -> "Credentials": + if params.get("api_key"): + raise _bad_request(_SIGV4_ONLY_MESSAGE) + auth: Final = AwsAuthParams.model_validate(params) + if bool(auth.aws_access_key_id) != bool(auth.aws_secret_access_key): + raise _unauthenticated("aws_access_key_id and aws_secret_access_key must be set together") + ensure_optional_import("botocore") + from botocore.exceptions import ( + CredentialRetrievalError, + NoCredentialsError, + PartialCredentialsError, + ProfileNotFound, + ) + + try: + return _usable_credentials(self._aws.resolve_credentials(auth, region)) + except CredentialRetrievalError as error: + verbose_logger.warning("OpenSearch vector store AWS credential source failed: %s", error) + raise _unauthenticated("the AWS credential source failed, see the LiteLLM logs") from None + except (NoCredentialsError, PartialCredentialsError, ProfileNotFound) as error: + raise _unauthenticated(str(error)) from None + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + parsed: Final = urlsplit(api_base or "") + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise _bad_request( + "api_base must be the OpenSearch endpoint URL, for example " + "https://..aoss.amazonaws.com or https://." + ) + if _ACCOUNT_ID_LABEL.match(parsed.hostname) and aws_signing_target(api_base or "")[0] == "aoss": + raise _bad_request(_PER_ACCOUNT_MESSAGE) + return (api_base or "").rstrip("/") + + def get_vector_store_url(self, api_base: str, encoded_vector_store_id: str) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + def get_vector_store_list_url(self, api_base: str) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + @staticmethod + def _params( + litellm_params: Mapping[str, object], + optional_params: VectorStoreSearchOptionalRequestParams, + extra_body: Mapping[str, object] | None, + ) -> tuple[_OpenSearchParams, int]: + unknown: Final = sorted( + key for key in litellm_params if key.startswith(_PARAM_PREFIX) and key not in _KNOWN_PARAMS + ) + if unknown: + raise _bad_request( + f"Unknown OpenSearch vector store parameter(s): {', '.join(unknown)}. " + f"Supported: {', '.join(sorted(_KNOWN_PARAMS))}." + ) + if extra_body: + raise _bad_request("The OpenSearch vector store does not support extra_body.") + try: + options: Final = _SearchOptions.model_validate(optional_params) + except ValidationError as error: + raise _bad_request(f"Invalid vector store search parameters: {error}") from None + if options.unsupported and not drops_unsupported_params(litellm_params.get("drop_params")): + raise _bad_request( + f"The OpenSearch vector store does not support {', '.join(options.unsupported)} yet. " + "Remove it from the request, or set drop_params to drop it." + ) + try: + return _OpenSearchParams.model_validate(litellm_params), options.k + except ValidationError: + raise _bad_request( + "opensearch_embedding_field, opensearch_index_name, opensearch_text_field, and litellm_embedding_model " + "must be strings." + ) from None + + @staticmethod + def _search_request( + vector_store_id: str, + embedding: EmbeddingResponse, + k: int, + params: _OpenSearchParams, + api_base: str, + ) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body + vector: Final = _QUERY_VECTOR.validate_python(embedding.data[0]["embedding"]) # pyright: ignore[reportUnknownMemberType] # EmbeddingResponse.data is an untyped list, the vector is validated here + return ( + f"{api_base}/{quote(params.index_name(vector_store_id), safe='')}/_search", + { + "size": k, + "query": {"knn": {params.embedding_field: {"vector": vector, "k": k}}}, + "_source": {"excludes": [params.embedding_field]}, + }, + ) + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: "LiteLLMLoggingObj", + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body + params, k = self._params(litellm_params, vector_store_search_optional_params, extra_body) + embedding: Final = self.query_embedding_executor(embedding_executor, None).embed( + params.require_embedding_model(), self.query_text(query), self.query_embedding_configuration(litellm_params) + ) + return self._search_request(vector_store_id, embedding, k, params, api_base) + + async def atransform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: "LiteLLMLoggingObj", + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body + params, k = self._params(litellm_params, vector_store_search_optional_params, extra_body) + embedding: Final = await self.query_embedding_executor(embedding_executor, None).aembed( + params.require_embedding_model(), self.query_text(query), self.query_embedding_configuration(litellm_params) + ) + return self._search_request(vector_store_id, embedding, k, params, api_base) + + def sign_request( + self, + headers: Mapping[str, str], + optional_params: Mapping[str, object], + request_data: Mapping[str, object], + api_base: str, + api_key: str | None = None, + ) -> tuple[dict[str, str], bytes]: # mutable-ok: the shared HTTP handler requires writable headers + region: Final = self._signing_region(optional_params, api_base) + body: Final = json.dumps(request_data) + signed: Final = sign_aws_json_post( + get_credentials=partial(self._signing_credentials, optional_params, region), + service_name=aws_signing_target(api_base)[0], + aws_region_name=region, + url=api_base, + body=body, + headers={ + "Content-Type": "application/json", + "x-amz-content-sha256": hashlib.sha256(body.encode()).hexdigest(), + }, + ) + signed_headers: Final = dict(signed.headers.items()) + signed_names: Final = frozenset(name.lower() for name in signed_headers) + return { + **{name: value for name, value in headers.items() if name.lower() not in signed_names}, + **signed_headers, + }, body.encode() + + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: "LiteLLMLoggingObj" + ) -> VectorStoreSearchResponse: + if not response.is_success: + raise BaseLLMException(status_code=response.status_code, message=response.text, headers=response.headers) + try: + hits: Final = _SearchResponse.model_validate_json(response.content).hits.hits + except ValidationError: + raise BaseLLMException( + status_code=502, + message="OpenSearch answered the search without a hits.hits list.", + headers=response.headers, + ) from None + logged: Final = _LoggedSearch.model_validate(litellm_logging_obj.model_call_details) + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query=logged.search_query, + data=[_search_result(hit, logged.litellm_params) for hit in hits], + ) + + def validate_create_vector_store(self) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + def transform_create_vector_store_request( + self, vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, api_base: str + ) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) + + def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn: + raise _bad_request(_SEARCH_ONLY_MESSAGE) diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 622c28f3c90..0270f814328 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import BaseModel import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import message_with_file_parts_as_image_urls from litellm.litellm_core_utils.url_utils import join_api_base from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -104,7 +105,7 @@ def _message_role_mapping(role: str) -> type[ChatMessage]: def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: - folded_message = _fold_message_cache_control(message) + folded_message = _fold_message_cache_control(message_with_file_parts_as_image_urls(message)) message_role_class = _message_role_mapping(str(folded_message["role"])) validated_message = validate_dict(folded_message, message_role_class) template.append(validated_message) diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 030413030eb..928dcc3bc52 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -112,7 +112,7 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: pdf/text data URIs become document blocks rather than malformed image blocks. """ image_url: Final = block.get("image_url") - url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url") + url: Final = image_url if isinstance(image_url, str) else _str_field(image_url, "url") if not url: return block @@ -121,18 +121,31 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document" else create_anthropic_image_param( image_url if isinstance(image_url, dict) else url, - format=_image_url_field(image_url, "format"), + format=_str_field(image_url, "format"), is_bedrock_invoke=True, ) ) + return _with_cache_control(converted, block) + + +def _convert_file_to_anthropic(block: Mapping[str, object]) -> object: + file_data: Final = _str_field(block.get("file"), "file_data") + if not file_data: + return block + return _with_cache_control( + anthropic_process_openai_file_message({"type": "file", "file": {"file_data": file_data}}), block + ) + + +def _with_cache_control(converted: Mapping[str, object], block: Mapping[str, object]) -> Mapping[str, object]: cache_control: Final = block.get("cache_control") if cache_control is None: return converted return {**converted, "cache_control": cache_control} -def _image_url_field(image_url: object, key: str) -> str | None: - value: Final = image_url.get(key) if isinstance(image_url, dict) else None +def _str_field(container: object, key: str) -> str | None: + value: Final = container.get(key) if isinstance(container, dict) else None return value if isinstance(value, str) else None @@ -141,15 +154,23 @@ def _data_uri_media_type(url: str) -> str: return match.group(1) if match else "" -def _convert_image_url_blocks_to_anthropic(content: object) -> object: +def _convert_media_block_to_anthropic(block: object) -> object: + mapping: Final = as_str_mapping(block) + if mapping is None: + return block + match mapping.get("type"): + case "image_url": + return _convert_image_url_to_anthropic(mapping) + case "file": + return _convert_file_to_anthropic(mapping) + case _: + return block + + +def _convert_media_blocks_to_anthropic(content: object) -> object: if not isinstance(content, list): return content - return [ - _convert_image_url_to_anthropic(block) - if isinstance(block, Mapping) and block.get("type") == "image_url" - else block - for block in content - ] + return [_convert_media_block_to_anthropic(block) for block in content] def _convert_tool_result_to_anthropic( @@ -391,7 +412,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): conversation.append( { "role": role, - "content": _convert_image_url_blocks_to_anthropic(content), + "content": _convert_media_blocks_to_anthropic(content), } ) diff --git a/litellm/llms/soniox/common_utils.py b/litellm/llms/soniox/common_utils.py index 8d83b4c9218..796f6ca12e4 100644 --- a/litellm/llms/soniox/common_utils.py +++ b/litellm/llms/soniox/common_utils.py @@ -3,7 +3,7 @@ Shared utilities for the Soniox provider (https://soniox.com). """ from collections.abc import Mapping, Sequence -from typing import Final, TypeAlias +from typing import Final, Literal, TypeAlias from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SubtitleToken, @@ -17,6 +17,8 @@ SonioxToken: TypeAlias = Mapping[str, object] # Soniox API base URL. SONIOX_API_BASE: Final[str] = "https://api.soniox.com" +SONIOX_REALTIME_MODEL_PREFIX: Final = "stt-rt-" + # Default polling interval in seconds when waiting for an async transcription # to finish. Mirrors the Soniox SDK default. SONIOX_DEFAULT_POLL_INTERVAL: Final[float] = 1.0 @@ -54,6 +56,10 @@ class SonioxException(BaseLLMException): """Provider-specific exception class for Soniox.""" +def soniox_health_check_mode(model: str) -> Literal["realtime"] | None: + return "realtime" if model.startswith(SONIOX_REALTIME_MODEL_PREFIX) else None + + def get_soniox_api_key(api_key: str | None = None) -> str | None: """Resolve the Soniox API key from arg or env var.""" # Local import to avoid a circular import: litellm.secret_managers.main diff --git a/litellm/llms/soniox/realtime/__init__.py b/litellm/llms/soniox/realtime/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py new file mode 100644 index 00000000000..ed7590b6f39 --- /dev/null +++ b/litellm/llms/soniox/realtime/transformation.py @@ -0,0 +1,609 @@ +import asyncio +import time +from collections import deque +from collections.abc import Awaitable, Callable, Mapping +from itertools import chain, groupby +from types import MappingProxyType, TracebackType +from typing import Final +from urllib.parse import urlparse, urlunparse + +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError +from typing_extensions import NotRequired, ReadOnly, Required, Self, TypedDict + +from litellm import verbose_logger +from litellm._uuid import uuid +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.realtime.transcription_protocol import ( + SESSION_UPDATE_EVENT_TYPES, + RealtimeTranscriptionProtocolError, + TranscriptionSessionUpdate, + completed_event, + decode_pcm16_append, + delta_event, + duration_usage, + error_event, + json_object, + parse_transcription_session_update, + speech_event, + transcription_session, + transcription_session_created_event, + uses_server_vad, +) +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend +from litellm.llms.custom_httpx.http_handler import realtime_ssl_for_url +from litellm.llms.soniox.common_utils import SonioxException, get_soniox_api_key +from litellm.types.llms.openai import ( + OpenAIRealtimeErrorEvent, + OpenAIRealtimeEvents, + OpenAIRealtimeInputAudioTranscriptionCompleted, + OpenAIRealtimeInputAudioTranscriptionDelta, + OpenAIRealtimeTranscriptionSessionCreated, +) +from litellm.types.realtime import ( + RealtimeInputAudioTranscriptionUsage, + RealtimeResponseTransformInput, + RealtimeResponseTypedDict, +) + +DEFAULT_SONIOX_REALTIME_URL: Final = "wss://stt-rt.soniox.com/transcribe-websocket" +WEBSOCKET_SCHEMES: Final = MappingProxyType({"wss": "wss", "ws": "ws", "https": "wss", "http": "ws"}) +DEFAULT_SAMPLE_RATE: Final = 24_000 +KEEPALIVE_INTERVAL_SECONDS: Final = 5.0 +_MAX_PENDING_FRAMES: Final = 64 +_FINALIZE: Final = '{"type":"finalize"}' +_KEEPALIVE: Final = '{"type":"keepalive"}' +_END_STREAM: Final = "" +_HEALTH_CHECK_SILENCE: Final = bytes(DEFAULT_SAMPLE_RATE // 5) +_CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) +SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' +_ENDPOINT_TOKEN: Final = "" +_FINALIZED_TOKEN: Final = "" + + +class SonioxProtocolError(RealtimeTranscriptionProtocolError): + pass + + +class SonioxRealtimeOptions(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + language_hints: tuple[str, ...] | None = None + language_hints_strict: bool | None = None + enable_language_identification: bool | None = None + enable_speaker_diarization: bool | None = None + context: JsonValue | None = None + translation: JsonValue | None = None + client_reference_id: str | None = None + max_endpoint_delay_ms: int | None = None + endpoint_sensitivity: float | None = None + endpoint_latency_adjustment_level: int | None = None + + +class SonioxStartRequest(SonioxRealtimeOptions): + model: str + audio_format: str + sample_rate: int + num_channels: int + enable_endpoint_detection: bool + + +class SonioxSpeechLabels(TypedDict, total=False): + speaker: ReadOnly[str] + language: ReadOnly[str] + + +class SonioxTranscriptSegment(SonioxSpeechLabels): + text: Required[ReadOnly[str]] + + +class SonioxTranscriptionDelta(OpenAIRealtimeInputAudioTranscriptionDelta, SonioxSpeechLabels): + pass + + +class SonioxTranscriptionCompleted(OpenAIRealtimeInputAudioTranscriptionCompleted): + segments: NotRequired[ReadOnly[tuple[SonioxTranscriptSegment, ...]]] + + +class _SonioxToken(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + is_final: bool = False + translation_status: str | None = None + speaker: str | None = None + language: str | None = None + + @property + def labelled(self) -> bool: + return self.speaker is not None or self.language is not None + + +class _SonioxResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + tokens: tuple[_SonioxToken, ...] = () + total_audio_proc_ms: int | None = None + error_code: int | None = None + error_message: str | None = None + finished: bool = False + + +def _sample_rate(update: TranscriptionSessionUpdate | None) -> int: + audio_format: Final = None if update is None else update.audio_format + if audio_format is None: + return DEFAULT_SAMPLE_RATE + if not audio_format.is_pcm16: + raise SonioxProtocolError("Soniox realtime requires pcm16 input audio") + if audio_format.channels not in (None, 1): + raise SonioxProtocolError("Soniox realtime requires mono input audio") + if audio_format.rate is None: + return DEFAULT_SAMPLE_RATE + if audio_format.rate <= 0: + raise SonioxProtocolError("sample rate must be positive") + return audio_format.rate + + +def _language_hints( + language: str | None, configured: tuple[str, ...] | None, strict: bool | None +) -> tuple[str, ...] | None: + if not language or (strict and configured): + return configured + return (language, *(hint for hint in configured or () if hint != language)) + + +def _uses_endpoint_detection(update: TranscriptionSessionUpdate, drop_params: bool) -> bool: + return uses_server_vad(update, "Soniox realtime", drop_params, SonioxProtocolError) + + +def _ignored_settings(update: TranscriptionSessionUpdate) -> tuple[str, ...]: + vad_tuning: Final = (f"turn_detection.{key}" for key in sorted(update.turn_detection or {}) if key != "type") + return (*update.unsupported_transcription_keys, *vad_tuning) + + +def build_start_request( + model: str, update: TranscriptionSessionUpdate | None, options: SonioxRealtimeOptions, drop_params: bool = False +) -> SonioxStartRequest: + return SonioxStartRequest( + **options.model_dump(exclude={"language_hints"}), + language_hints=_language_hints( + None if update is None else update.language, options.language_hints, options.language_hints_strict + ), + model=model, + audio_format="pcm_s16le", + sample_rate=_sample_rate(update), + num_channels=1, + enable_endpoint_detection=update is None or _uses_endpoint_detection(update, drop_params), + ) + + +def _session_created(session_id: str, start_request: SonioxStartRequest) -> OpenAIRealtimeTranscriptionSessionCreated: + session: Final = transcription_session( + session_id=session_id, + model=start_request.model, + sample_rate=start_request.sample_rate, + language=start_request.language_hints[0] if start_request.language_hints else None, + server_vad=start_request.enable_endpoint_detection, + ) + return transcription_session_created_event(session) + + +def _changes_started_rate(update: TranscriptionSessionUpdate, started: SonioxStartRequest) -> bool: + audio_format: Final = update.audio_format + if audio_format is None: + return False + requested_rate: Final = _sample_rate(update) + return audio_format.rate is not None and requested_rate != started.sample_rate + + +def _changes_started_stream(update: TranscriptionSessionUpdate, started: SonioxStartRequest, drop_params: bool) -> bool: + turn_detection_set: Final = update.turn_detection is not None or update.turn_detection_disabled + return ( + _changes_started_rate(update, started) + or ( + update.language is not None + and _language_hints(update.language, started.language_hints, started.language_hints_strict) + != started.language_hints + ) + or (turn_detection_set and _uses_endpoint_detection(update, drop_params) != started.enable_endpoint_detection) + ) + + +def build_soniox_realtime_url(api_base: str | None) -> str: + if api_base is None: + return DEFAULT_SONIOX_REALTIME_URL + parsed: Final = urlparse(api_base) + scheme: Final = WEBSOCKET_SCHEMES.get(parsed.scheme) + if scheme is None: + raise ValueError("Soniox realtime api_base must be a wss://, ws://, https://, or http:// URL") + return urlunparse(parsed._replace(scheme=scheme)) + + +def _labels(token: _SonioxToken) -> SonioxSpeechLabels: + speaker: Final[SonioxSpeechLabels] = {} if token.speaker is None else {"speaker": token.speaker} + language: Final[SonioxSpeechLabels] = {} if token.language is None else {"language": token.language} + return {**speaker, **language} + + +def _delta(item_id: str, token: _SonioxToken) -> SonioxTranscriptionDelta: + return {**delta_event(item_id, token.text), **_labels(token)} + + +def _segment(run: tuple[_SonioxToken, ...]) -> SonioxTranscriptSegment: + return {**_labels(run[0]), "text": "".join(token.text for token in run).strip()} + + +def _segments(tokens: tuple[_SonioxToken, ...]) -> tuple[SonioxTranscriptSegment, ...]: + return tuple(_segment(tuple(run)) for _, run in groupby(tokens, key=lambda token: (token.speaker, token.language))) + + +def _new_item_id() -> str: + return f"item_{uuid.uuid4().hex}" + + +def _completed( + item_id: str, tokens: tuple[_SonioxToken, ...], usage: RealtimeInputAudioTranscriptionUsage | None +) -> OpenAIRealtimeInputAudioTranscriptionCompleted: + completed: Final = completed_event(item_id, "".join(token.text for token in tokens).strip(), usage) + if not any(token.labelled for token in tokens): + return completed + labelled: Final[SonioxTranscriptionCompleted] = {**completed, "segments": _segments(tokens)} + return labelled + + +def _soniox_error(response: _SonioxResponse) -> SonioxException | None: + if response.error_code is None and response.error_message is None: + return None + return SonioxException( + status_code=response.error_code or 500, + message=f"Soniox realtime error {response.error_code}: {response.error_message}", + ) + + +class SonioxEventTransformer: + def __init__(self, *, drops_originals: bool) -> None: + self._drops_originals: Final = drops_originals + self._item_id: str | None = None + self._final_tokens: Final[list[_SonioxToken]] = [] # mutable-ok: a turn's tokens append in O(1) each + self._finalizes: Final[deque[bool]] = deque() # mutable-ok: pending finalizes queue and pop in O(1) each + self._oldest_commit_answered: bool = False + self._processed_ms: int = 0 + self._billed_ms: int = 0 + self._refusal: SonioxException | None = None + + @property + def refusal(self) -> SonioxException | None: + return self._refusal + + def transform(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + try: + response: Final = _SonioxResponse.model_validate_json(payload) + except ValidationError: + raise SonioxProtocolError("Soniox returned an invalid realtime response") from None + self._bill(response.total_audio_proc_ms) + error: Final = _soniox_error(response) + if error is not None: + return (self._report(error),) + return tuple(chain.from_iterable(self._token_events(token) for token in response.tokens)) + + def queue_finalize(self, *, discards_turn: bool) -> None: + if discards_turn and not self._finalizes: + self._item_id = None + self._final_tokens.clear() + self._finalizes.append(discards_turn) + + def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None: + billable_ms: Final = max(self._processed_ms, stream_ms) + unbilled_ms: Final = billable_ms - self._billed_ms + if unbilled_ms <= 0: + return None + self._billed_ms = billable_ms + return duration_usage(unbilled_ms / 1000) + + def _report(self, error: SonioxException) -> OpenAIRealtimeErrorEvent: + if self._processed_ms == 0: + self._refusal = error + return error_event(error.message) + + def _bill(self, total_audio_proc_ms: int | None) -> None: + if total_audio_proc_ms is not None and total_audio_proc_ms > self._processed_ms: + self._processed_ms = total_audio_proc_ms + + def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]: + if token.text == _FINALIZED_TOKEN: + return self._finalized() + if self._discarding: + return () + if token.text == _ENDPOINT_TOKEN: + return self._end_turn() + started: Final = self._start() + if not token.is_final or (self._drops_originals and token.translation_status == "original"): + return started + self._final_tokens.append(token) + return (*started, _delta(self._current_item_id(), token)) + + @property + def _discarding(self) -> bool: + return bool(self._finalizes) and self._finalizes[0] + + def _finalized(self) -> tuple[OpenAIRealtimeEvents, ...]: + discarded: Final = self._finalizes.popleft() if self._finalizes else None + answered: Final = self._oldest_commit_answered + self._oldest_commit_answered = False + if discarded: + return () + if self._item_id is not None: + return self._complete(speech_stopped=False) + if discarded is None or answered: + return () + return (_completed(_new_item_id(), (), self.take_unbilled_usage()),) + + def _end_turn(self) -> tuple[OpenAIRealtimeEvents, ...]: + events: Final = self._complete(speech_stopped=True) + if events and self._finalizes: + self._oldest_commit_answered = True + return events + + def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: + if self._item_id is not None: + return () + item_id: Final = _new_item_id() + self._item_id = item_id + return (speech_event("input_audio_buffer.speech_started", item_id),) + + def _current_item_id(self) -> str: + assert self._item_id is not None + return self._item_id + + def _complete(self, *, speech_stopped: bool) -> tuple[OpenAIRealtimeEvents, ...]: + item_id: Final = self._item_id + if item_id is None: + return () + tokens: Final = tuple(self._final_tokens) + self._item_id = None + self._final_tokens.clear() + stopped: Final[tuple[OpenAIRealtimeEvents, ...]] = ( + (speech_event("input_audio_buffer.speech_stopped", item_id),) if speech_stopped else () + ) + return (*stopped, _completed(item_id, tokens, self.take_unbilled_usage())) + + +class SonioxRealtimeBackend: + def __init__( + self, + inner: RealtimeBackend, + *, + keepalive_interval: float = KEEPALIVE_INTERVAL_SECONDS, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._inner: Final = inner + self._interval: Final = keepalive_interval + self._clock: Final = clock + self._sleep: Final = sleep + self._frames: Final[asyncio.Queue[str | bytes | Exception]] = asyncio.Queue(maxsize=_MAX_PENDING_FRAMES) + self._last_sent: float = clock() + self._started: bool = False + self._ended: bool = False + self._tasks: tuple[asyncio.Task[None], ...] = () + + async def __aenter__(self) -> Self: + await self._inner.__aenter__() + self._tasks = (asyncio.create_task(self._pump()), asyncio.create_task(self._keepalive())) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._stop() + await self._inner.__aexit__(exc_type, exc_value, traceback) + + async def send(self, message: str | bytes) -> None: + self._last_sent = self._clock() + self._ended = self._ended or message == _END_STREAM + await self._inner.send(message) + if not self._started and isinstance(message, str) and message not in _CONTROL_FRAMES: + self._started = True + await self._frames.put(SESSION_STARTED_FRAME) + + async def recv(self, decode: bool | None = None) -> str | bytes: + frame: Final = await self._frames.get() + if isinstance(frame, Exception): + raise frame + return frame + + async def close(self) -> None: + self._stop() + await self._inner.close() + + def _stop(self) -> None: + for task in self._tasks: + task.cancel() + + async def _pump(self) -> None: + while True: + try: + frame = await self._inner.recv() + except Exception as e: # noqa: BLE001 # recv() re-raises it, so an upstream failure never hangs the reader + await self._frames.put(e) + return + await self._frames.put(frame) + + async def _keepalive(self) -> None: + while not self._ended: + await self._sleep(self._interval) + if self._ended or self._clock() - self._last_sent < self._interval: + continue + try: + await self.send(_KEEPALIVE) + except Exception as e: # noqa: BLE001 # stops only keepalives; the pump reports the broken socket + verbose_logger.debug("Soniox realtime: stopping keepalive after send failed: %s", e) + return + + +class SonioxRealtimeConfig(BaseRealtimeConfig): + def __init__( + self, + *, + options: SonioxRealtimeOptions | None = None, + wrap: Callable[[RealtimeBackend], RealtimeBackend] = SonioxRealtimeBackend, + clock: Callable[[], float] = time.monotonic, + drop_params: bool = False, + ) -> None: + self._options: Final = options or SonioxRealtimeOptions() + self._drop_params: Final = drop_params + self._wrap: Final = wrap + self._clock: Final = clock + self._transformer: Final = SonioxEventTransformer(drops_originals=self._options.translation is not None) + self._start_request: SonioxStartRequest | None = None + self._session_id: str = f"sess_{uuid.uuid4().hex}" + self._started_at: float | None = None + + @classmethod + def from_litellm_params(cls, litellm_params: Mapping[str, object], *, drop_params: bool = False) -> Self: + return cls(options=SonioxRealtimeOptions.model_validate(dict(litellm_params)), drop_params=drop_params) + + def requires_session_configuration(self) -> bool: + return True + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseRealtimeConfig contract + model: str, + api_key: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseRealtimeConfig contract + resolved_key: Final = get_soniox_api_key(api_key) + if not resolved_key: + raise ValueError("Missing Soniox API key. Set SONIOX_API_KEY or api_key in the deployment litellm_params") + return {**headers, "Authorization": f"Bearer {resolved_key}"} + + def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str: + return build_soniox_realtime_url(api_base) + + def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + return self._wrap(backend) + + def transform_session_created_event( + self, + model: str, + logging_session_id: str, + session_configuration_request: str | None = None, + ) -> OpenAIRealtimeTranscriptionSessionCreated: + self._session_id = logging_session_id + return _session_created(logging_session_id, build_start_request(model, None, self._options)) + + def transform_realtime_request( + self, + message: str, + model: str, + session_configuration_request: str | None = None, + ) -> tuple[str | bytes, ...]: + request: Final = json_object(message, SonioxProtocolError) + event_type: Final = request.get("type") + if event_type is None: + raise SonioxProtocolError("Soniox realtime takes OpenAI realtime events, and this message has no type") + if event_type in SESSION_UPDATE_EVENT_TYPES: + update: Final = parse_transcription_session_update(message, SonioxProtocolError) + ignored: Final = _ignored_settings(update) + if ignored: + verbose_logger.warning("Soniox realtime: ignoring unsupported session settings %s", ignored) + return self._start(model, update) + if event_type == "input_audio_buffer.append": + audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError) + return (*self._start(model, None), audio) + if event_type == "input_audio_buffer.commit": + return self._finalize(model, discards_turn=False) + if event_type == "input_audio_buffer.clear": + return () if self._start_request is None else self._finalize(model, discards_turn=True) + if event_type == "input_audio_buffer.end": + return (*self._start(model, None), _END_STREAM) + verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) + return () + + def _finalize(self, model: str, *, discards_turn: bool) -> tuple[str, ...]: + start: Final = self._start(model, None) + self._transformer.queue_finalize(discards_turn=discards_turn) + return (*start, _FINALIZE) + + def session_refusal(self) -> Exception | None: + return self._transformer.refusal + + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: + started_at: Final = self._started_at + if self._transformer.refusal is not None: + return None + if started_at is None: + return self._transformer.take_unbilled_usage() + return self._transformer.take_unbilled_usage(int((self._clock() - started_at) * 1000)) + + def transform_realtime_response( + self, + message: str | bytes, + model: str, + logging_obj: LiteLLMLoggingObj, + realtime_response_transform_input: RealtimeResponseTransformInput, + ) -> RealtimeResponseTypedDict: + payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message + result: Final[RealtimeResponseTypedDict] = { + "response": list(self._backend_events(payload)), + "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), + "current_response_id": realtime_response_transform_input.get("current_response_id"), + "current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"), + "current_conversation_id": realtime_response_transform_input.get("current_conversation_id"), + "current_item_chunks": realtime_response_transform_input.get("current_item_chunks"), + "current_delta_type": realtime_response_transform_input.get("current_delta_type"), + "session_configuration_request": realtime_response_transform_input.get("session_configuration_request"), + } + return result + + def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + if payload != SESSION_STARTED_FRAME: + return self._transformer.transform(payload) + assert self._start_request is not None + return (_session_created(self._session_id, self._start_request),) + + def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]: + started: Final = self._start_request + if started is not None: + if update is not None and _changes_started_stream(update, started, self._drop_params): + raise SonioxProtocolError("Soniox realtime can't change the session once the stream has started") + return () + start_request: Final = build_start_request(model, update, self._options, self._drop_params) + self._start_request = start_request + self._started_at = self._clock() + return (start_request.model_dump_json(exclude_none=True),) + + +def _finishes_stream(frame: str | bytes) -> bool: + response: Final = _SonioxResponse.model_validate_json(frame) + error: Final = _soniox_error(response) + if error is not None: + raise error + return response.finished + + +async def soniox_realtime_health_check( + model: str, api_base: str | None, api_key: str | None, options: SonioxRealtimeOptions +) -> bool: + import websockets + + config: Final = SonioxRealtimeConfig() + url: Final = config.get_complete_url(api_base, model) + headers: Final = config.validate_environment({}, model, api_key) + start_request: Final = build_start_request(model, None, options) + async with websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=realtime_ssl_for_url(url), + ) as backend: + await backend.send(start_request.model_dump_json(exclude_none=True)) + await backend.send(_HEALTH_CHECK_SILENCE) + await backend.send(_END_STREAM) + async for frame in backend: + if _finishes_stream(frame): + return True + raise SonioxException(status_code=502, message="Soniox closed the realtime stream without finishing it") diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 5bff43828d6..884c82fe331 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from dataclasses import dataclass from typing import Final, Literal import httpx @@ -8,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import Cache, LiteLLMCacheType -from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -16,10 +16,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.llms.openai.openai import AllMessageValues -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.types.llms.openai import ChatCompletionToolParam +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url -from ..vertex_llm_base import VertexBase +from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, @@ -65,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) +_TOOL_PARAMS: Final = ("tools", "tool_choice") + + +def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler: + if timeout is None: + return HTTPHandler() + return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout) + + +class _CachedContentsNotServedError(VertexAIError): + pass + + def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None: tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0 logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count @@ -78,6 +92,61 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: return timeout +@dataclass(frozen=True, slots=True) +class _CacheTarget: + model: str + project: str | None + location: str | None + + +def _cache_target( + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + api_base: str | None, + model: str, + vertex_project: str | None, + vertex_location: str | None, +) -> _CacheTarget: + model_location: Final = ( + None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base) + ) + if model_location is None: + return _CacheTarget(model=model, project=vertex_project, location=vertex_location) + return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location) + + +def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int: + return max( + get_prompt_cache_min_tokens( + model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}" + ) + for model in models + ) + + +def _context_caching_skip_reason( + cached_messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam] | None, + deployment_model: str, + cache_model: str, + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], +) -> str | None: + if not cached_messages_end_on_supported_turn(cached_messages): + return ( + "Vertex AI context caching: cached message block ends on a model turn once " + "system messages are extracted, which the cachedContents API rejects." + ) + min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider) + if not is_prompt_caching_valid_prompt( + model=cache_model, + messages=list(cached_messages), + tools=None if tools is None else list(tools), + custom_llm_provider=custom_llm_provider, + min_token_count=min_tokens, + ): + return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})." + return None + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -135,6 +204,7 @@ class ContextCachingEndpoints(VertexBase): vertex_project=vertex_project, vertex_location=vertex_location, vertex_api_version=("v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1"), + collection_endpoint=True, ) def check_cache( @@ -206,6 +276,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -305,6 +377,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -370,41 +444,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -417,44 +475,42 @@ class ContextCachingEndpoints(VertexBase): if extra_headers is not None: headers.update(extra_headers) - if client is None or not isinstance(client, HTTPHandler): - _params: Final = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = HTTPHandler(**_params) - else: - client = client + http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout) ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model - ) - google_cache_name: Final = self.check_cache( - cache_key=generated_cache_key, - client=client, - headers=headers, - api_key=api_key, - api_base=api_base, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_auth_header=vertex_auth_header, - model=model, + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) + try: + google_cache_name: Final = self.check_cache( + cache_key=generated_cache_key, + client=http_client, + headers=headers, + api_key=api_key, + api_base=api_base, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vertex_project=target.project, + vertex_location=target.location, + vertex_auth_header=vertex_auth_header, + model=target.model, + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) if google_cache_name: return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -473,7 +529,7 @@ class ContextCachingEndpoints(VertexBase): ) try: - response: Final = client.post( + response: Final = http_client.post( url=url, headers=headers, json=cached_content_request_body, @@ -489,7 +545,7 @@ class ContextCachingEndpoints(VertexBase): _record_cache_creation_tokens(logging_obj, raw_response_cached) self._known_caches.remember( KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ), generated_cache_key, raw_response_cached.get("name"), @@ -536,41 +592,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -590,10 +630,10 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) scope: Final = KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ) async def lookup_or_create() -> str | None: @@ -605,10 +645,10 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) if google_cache_name: @@ -616,12 +656,12 @@ class ContextCachingEndpoints(VertexBase): ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -659,9 +699,15 @@ class ContextCachingEndpoints(VertexBase): ) return raw_response_cached.get("name") - cache_name: Final = await self._known_caches.single_flight( - scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) - ) + try: + cache_name: Final = await self._known_caches.single_flight( + scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name def get_cache(self): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..bfa94396ea7 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -211,6 +211,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + beta_values.update( + anthropic_model_info.thinking_betas( + optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict + ) + ) + if beta_values: request_headers["anthropic-beta"] = ",".join(beta_values) @@ -245,7 +251,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 363e5596cb9..79fd9f9e8b1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -145,6 +145,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="vertex_ai", + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) beta_set: Final = set(auto_betas) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 4be41e26326..180761f15d9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,9 +6,12 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Mapping from typing import Any, Final +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -20,6 +23,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +def count_tokens_request_body( + request_data: Mapping[str, object], model: str, drop_params: object +) -> Mapping[str, object]: + tools: Final = request_data.get("tools") + if not isinstance(tools, list): + return request_data + return { + **request_data, + "tools": tools_without_custom_field( + OBJECT_LIST_ADAPTER.validate_python(tools), + hoist_defer_loading=is_claude_model_target("vertex_ai", model), + drop_custom=drops_unsupported_params(drop_params), + ), + } + + def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) @@ -169,7 +188,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): response: Final = await async_client.post( endpoint_url, headers=headers, - json=request_data, + json=dict(count_tokens_request_body(request_data, model, litellm_params.get("drop_params"))), timeout=30.0, ) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index bb200fd542d..a3a3427670d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -9,6 +9,7 @@ import httpx import litellm from litellm import LlmProviders +from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls from litellm.types.llms.vertex_ai import VertexPartnerProvider from litellm.utils import ModelResponse @@ -217,7 +218,7 @@ class VertexAIPartnerModels(VertexBase): return base_llm_http_handler.completion( model=model, stream=stream, - messages=messages, + messages=list(messages_with_file_parts_as_image_urls(messages)), acompletion=acompletion, api_base=api_base, model_response=model_response, diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 0cdd92bb071..7dd98212f76 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -6,9 +6,11 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import asyncio import json +import re import threading from collections import OrderedDict from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse @@ -46,6 +48,31 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str: _graft_default_vertex_path = graft_default_vertex_path + +_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)") + + +@dataclass(frozen=True, slots=True) +class PublisherModelLocation: + parent_url: str + project: str + location: str + model: str + + +def publisher_model_location(api_base: str) -> PublisherModelLocation | None: + match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base) + if match is None: + return None + _, _, project, _, location = match.group().rsplit("/", 4) + return PublisherModelLocation( + parent_url=match.group(), + project=project, + location=location, + model=api_base.rstrip("/").rsplit("/", 1)[-1], + ) + + GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform" ) @@ -637,15 +664,18 @@ class VertexBase: vertex_location: str | None = None, vertex_api_version: Literal["v1", "v1beta1"] | None = None, use_psc_endpoint_format: bool = False, + collection_endpoint: bool = False, ) -> tuple[str | None, str]: """ for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317 Handles custom api_base for: - 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} + 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}, or {api_base}/{endpoint} + for a collection endpoint 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the api_base when its path is empty or only an API version (/v1, /v1beta1); - otherwise constructs {api_base}:{endpoint} + otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base + naming a publisher model, the collection under that model's projects/{project}/locations/{location} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -653,22 +683,22 @@ class VertexBase: Args: use_psc_endpoint_format: If True, constructs PSC endpoint URL format. If False (default), uses api_base as-is and appends :{endpoint} + collection_endpoint: True when `endpoint` names a collection resource such as + cachedContents rather than a model action such as generateContent ## Returns - (auth_header, url) - Tuple[Optional[str], str] """ if api_base: if custom_llm_provider == "gemini": - # For Gemini (Google AI Studio), construct the full path like other providers - if model is None: + if model is None and not collection_endpoint: raise ValueError("Model parameter is required for Gemini custom API base URLs") - url = join_api_base(api_base, f"models/{model}:{endpoint}") + url = join_api_base(api_base, endpoint if collection_endpoint else f"models/{model}:{endpoint}") if gemini_api_key is None: raise ValueError( "Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable." ) - if gemini_api_key is not None: - auth_header = {"x-goog-api-key": gemini_api_key} + auth_header = {"x-goog-api-key": gemini_api_key} else: # For Vertex AI if use_psc_endpoint_format: @@ -692,7 +722,12 @@ class VertexBase: elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: - url = f"{api_base.rstrip('/')}:{endpoint}" + model_location: Final = publisher_model_location(api_base) if collection_endpoint else None + url = ( + f"{api_base.rstrip('/')}:{endpoint}" + if model_location is None + else join_api_base(model_location.parent_url, endpoint) + ) if stream is True: parsed_stream_url: Final = urlparse(url) stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" diff --git a/litellm/main.py b/litellm/main.py index b8ad28c26fe..50ab274a060 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -108,6 +108,10 @@ from litellm.litellm_core_utils.mock_functions import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export ) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + messages_have_pdf_file_part, + messages_without_pdf_file_parts, +) from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, @@ -155,6 +159,7 @@ from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + AgenticLoopParams, BaseUrlRoutingHint, CustomPricingLiteLLMParams, ModelResponseStream, @@ -190,11 +195,13 @@ from litellm.utils import ( get_requester_metadata, get_secret, get_standard_openai_params, + is_explicitly_disabled_factory, mock_completion_streaming_obj, pre_process_non_default_params, read_config_args, should_run_mock_completion, supports_httpx_timeout, + supports_pdf_input, validate_and_fix_openai_messages, validate_and_fix_openai_tools, validate_and_fix_thinking_param, @@ -234,6 +241,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export ) from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.anthropic.common_utils import is_claude_tool_target from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -391,6 +399,125 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro return assert_never(control) +PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"}) +PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"}) + + +def _cost_map_pdf_input_support(model: str, custom_llm_provider: str) -> bool | None: + if custom_llm_provider in PROVIDERS_WITHOUT_DOCUMENT_INPUT or is_explicitly_disabled_factory( + model, custom_llm_provider, "supports_pdf_input" + ): + return False + if supports_pdf_input(model, custom_llm_provider): + return True + return None + + +def _deployment_pdf_input_support(model: str, base_model: str, custom_llm_provider: str) -> bool | None: + supports: Final = ( + _cost_map_pdf_input_support(model, custom_llm_provider), + _cost_map_pdf_input_support(base_model, custom_llm_provider), + ) + if False in supports: + return False + if True in supports: + return True + return None + + +def _declared_pdf_input_support( + model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None, base_model: str | None +) -> bool | None: + match model_info: + case {"supports_pdf_input": bool() as declared}: + return declared + case _ if base_model: + return _deployment_pdf_input_support(model, base_model, custom_llm_provider) + case _: + return _cost_map_pdf_input_support(model, custom_llm_provider) + + +def _pdf_input_unreadable( + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, +) -> bool: + match _declared_pdf_input_support(model, custom_llm_provider, model_info, base_model): + case False: + return True + case None: + return drop_params and custom_llm_provider in PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT + case True: + return False + + +def _messages_without_unreadable_pdfs( + messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, +) -> tuple[AllMessageValues, ...] | None: + if not messages_have_pdf_file_part(messages) or not _pdf_input_unreadable( + model, custom_llm_provider, model_info, base_model, drop_params + ): + return None + if not drop_params: + raise litellm.BadRequestError( + message=( + f"{custom_llm_provider}/{model} does not accept PDF input. " + "Remove the PDF, or set drop_params=True to send the request without it" + ), + model=model, + llm_provider=custom_llm_provider, + ) + readable_messages: Final = messages_without_pdf_file_parts(messages) + if not any(message["role"] == "user" for message in readable_messages): + raise litellm.BadRequestError( + message=( + f"{custom_llm_provider}/{model} does not accept PDF input, " + "and no user message is left once the PDF is dropped" + ), + model=model, + llm_provider=custom_llm_provider, + ) + return readable_messages + + +def _messages_readable_by_model( + messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, + ensure_alternating_roles: bool | None, + user_continue_message: ChatCompletionUserMessage | None, + assistant_continue_message: ChatCompletionAssistantMessage | None, +) -> tuple[AllMessageValues, ...] | None: + pdf_free_messages: Final = _messages_without_unreadable_pdfs( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + model_info=model_info, + base_model=base_model, + drop_params=drop_params, + ) + if pdf_free_messages is None: + return None + return tuple( + get_completion_messages( + messages=list(pdf_free_messages), + ensure_alternating_roles=ensure_alternating_roles or False, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + ) + + ####### COMPLETION ENDPOINTS ################ @@ -1135,17 +1262,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o _ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) -def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: - if custom_llm_provider == "anthropic": - return True - model_lower: Final = model.lower() - if custom_llm_provider == "bedrock": - return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) - if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": - return "claude" in model_lower - return False - - def _history_without_unanswered_tool_search_calls( messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig ) -> tuple[object, ...] | None: @@ -1282,6 +1398,16 @@ def _get_router_deployment_id(kwargs: dict) -> str | None: return None +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + def _register_custom_pricing_for_request( model: str, custom_llm_provider: str, @@ -4233,7 +4359,6 @@ def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchRe _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str]) -_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) @@ -5557,9 +5682,15 @@ def completion( ) headers = kwargs.get("headers", None) or extra_headers - ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None) - user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None) - assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None) + ensure_alternating_roles: Final = cast( # cast-ok: read from untyped kwargs + bool | None, kwargs.get("ensure_alternating_roles", None) + ) + user_continue_message: Final = cast( # cast-ok: read from untyped kwargs + ChatCompletionUserMessage | None, kwargs.get("user_continue_message", None) + ) + assistant_continue_message: Final = cast( # cast-ok: read from untyped kwargs + ChatCompletionAssistantMessage | None, kwargs.get("assistant_continue_message", None) + ) headers = {} if headers is None else dict(headers) if extra_headers is not None: headers.update(extra_headers) @@ -5701,6 +5832,7 @@ def completion( **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, "extra_headers": extra_headers, } + requested_model: Final = model model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5712,16 +5844,23 @@ def completion( else None ), ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5752,7 +5891,7 @@ def completion( } ) - if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): + if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) if provider_specific_header is not None: @@ -5818,7 +5957,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) @@ -5911,15 +6058,17 @@ def completion( def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) - optional_params = optional_params_for_model_info( + validated_model_info: Final = ( _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None ) + deployment_base_model: Final = base_model if isinstance(base_model, str) else None + optional_params = optional_params_for_model_info(validated_model_info) fakes_stream: Final = partial( _completion_fakes_stream, model, custom_llm_provider, provider_config, - base_model=base_model if isinstance(base_model, str) else None, + base_model=deployment_base_model, bridges_to_responses_api=bridges_to_responses_api is True, ) resolve_timeout: Final = partial( @@ -5957,16 +6106,29 @@ def completion( functions_unsupported_model: Final[list[object]] = TypeAdapter(list[object]).validate_python( optional_params.pop("functions_unsupported_model") ) - request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) messages = function_call_prompt( messages=messages, functions=functions_unsupported_model, model=model, llm_provider=custom_llm_provider, - drop_params=litellm.drop_params is True - or normalize_drop_params(request_kwargs.get("drop_params")) is True, + drop_params=drop_unsupported_reasoning, ) + readable_messages: Final = _messages_readable_by_model( + messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List + model=model, + custom_llm_provider=custom_llm_provider, + model_info=validated_model_info, + base_model=deployment_base_model, + drop_params=drop_unsupported_reasoning, + ensure_alternating_roles=ensure_alternating_roles, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + messages = ( # rebind-ok: completion() rewrites messages step by step + messages if readable_messages is None else list(readable_messages) + ) + # For logging - save the values of the litellm-specific params passed in requested_litellm_params: Final = get_litellm_params( acompletion=acompletion, diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 0e2fc2e3dcb..8e6becc60b4 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -111,6 +111,7 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, + Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field, } ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f36112bc0ef..dbe0be24fdc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4049,6 +4049,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true }, @@ -12437,7 +12438,8 @@ "supported_output_modalities": [ "text" ], - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_pdf_input": false }, "azure_ai/FLUX-1.1-pro": { "deprecation_date": "2027-01-05", @@ -12502,6 +12504,7 @@ "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12516,6 +12519,7 @@ "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12531,6 +12535,7 @@ "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12546,6 +12551,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12560,6 +12566,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12574,6 +12581,7 @@ "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12594,6 +12602,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12616,6 +12625,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12638,6 +12648,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12660,6 +12671,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12687,6 +12699,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12703,6 +12716,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12724,6 +12738,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12744,6 +12759,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12765,6 +12781,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12833,6 +12850,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12847,6 +12865,7 @@ "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { @@ -12859,6 +12878,7 @@ "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12872,6 +12892,7 @@ "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12884,6 +12905,7 @@ "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -12895,6 +12917,7 @@ "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -12906,6 +12929,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12918,6 +12942,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12930,6 +12955,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12942,6 +12968,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12954,6 +12981,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12966,6 +12994,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12978,6 +13007,7 @@ "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12990,6 +13020,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13002,6 +13033,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13015,6 +13047,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13027,7 +13060,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-multimodal-instruct": { "deprecation_date": "2027-01-17", @@ -13041,6 +13075,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_audio_input": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { @@ -13052,7 +13087,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-reasoning": { "deprecation_date": "2027-01-17", @@ -13064,6 +13100,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_reasoning": true }, @@ -13106,6 +13143,7 @@ "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/doc-intelligence/prebuilt-read": { @@ -13144,6 +13182,7 @@ "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -13202,6 +13241,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13217,6 +13257,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13230,6 +13271,7 @@ "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { @@ -13242,6 +13284,7 @@ "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 1.45e-07, @@ -13257,6 +13300,7 @@ "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.8e-08, @@ -13273,6 +13317,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13306,6 +13351,7 @@ "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13323,6 +13369,7 @@ "output_cost_per_token_above_200k_tokens": 5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13343,6 +13390,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13361,6 +13409,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13379,6 +13428,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13395,6 +13445,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13409,6 +13460,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13424,6 +13476,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13436,7 +13489,8 @@ "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false }, "azure_ai/jamba-instruct": { "deprecation_date": "2025-03-01", @@ -13447,6 +13501,7 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { @@ -13459,6 +13514,7 @@ "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true, @@ -13482,6 +13538,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, @@ -13498,6 +13555,7 @@ "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large": { @@ -13510,6 +13568,7 @@ "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-2407": { @@ -13522,6 +13581,7 @@ "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-latest": { @@ -13533,6 +13593,7 @@ "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-3": { @@ -13544,6 +13605,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13557,6 +13619,7 @@ "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-nemo": { @@ -13568,7 +13631,8 @@ "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/mistral-small": { "deprecation_date": "2025-07-31", @@ -13580,6 +13644,7 @@ "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-small-2503": { @@ -13591,6 +13656,7 @@ "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -52006,6 +52072,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52026,6 +52093,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52777,6 +52845,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_web_search": true }, @@ -52794,6 +52863,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -52811,6 +52881,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52989,6 +53060,7 @@ "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true }, "vertex_ai/openai/gpt-oss-20b-maas": { @@ -53000,6 +53072,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true, "cache_read_input_token_cost": 7e-09, "input_cost_per_token_batches": 3.5e-08, @@ -53131,6 +53204,7 @@ "us-south1" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { @@ -53149,6 +53223,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": { @@ -53164,6 +53239,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": { @@ -53179,6 +53255,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/veo-3.1-generate-preview": { @@ -61192,6 +61269,126 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview-v2": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v4": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", @@ -63670,6 +63867,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63816,6 +64014,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -67215,6 +67414,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67229,6 +67429,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67271,6 +67472,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67285,6 +67487,7 @@ "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67299,6 +67502,7 @@ "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67313,6 +67517,7 @@ "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67329,6 +67534,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -67344,6 +67550,7 @@ "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -67362,6 +67569,7 @@ "supported_output_modalities": [ "text" ], + "supports_pdf_input": false, "supports_response_schema": true, "supports_vision": true }, @@ -67371,7 +67579,8 @@ "output_cost_per_token": 5.4e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3-0324": { "deprecation_date": "2026-07-13", @@ -67379,7 +67588,8 @@ "output_cost_per_token": 4.56e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3.1": { "deprecation_date": "2026-07-13", @@ -67387,7 +67597,8 @@ "output_cost_per_token": 4.94e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3": { "deprecation_date": "2026-05-01", @@ -67395,7 +67606,8 @@ "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3-mini": { "deprecation_date": "2026-05-01", @@ -67403,7 +67615,8 @@ "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", @@ -67411,7 +67624,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-reasoning": { "deprecation_date": "2026-05-01", @@ -67419,7 +67633,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -73089,6 +73304,7 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": ["global"], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -73704,6 +73920,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -74758,6 +74975,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, @@ -75006,6 +75224,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://docs.x.ai/developers/models/grok-4.7", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -78354,6 +78573,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 7b0234e28d7..1d487f33835 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3724,7 +3724,7 @@ "ocr": false, "search": false, "moderations": false, - "realtime": false, + "realtime": true, "video_generations": false, "batches": false, "messages": false, @@ -4910,6 +4910,33 @@ "vector_stores_search": true } }, + "opensearch": { + "display_name": "Amazon OpenSearch (`opensearch`)", + "url": "https://docs.litellm.ai/docs/providers/opensearch_vector_stores", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_search": true + } + }, "helicone": { "display_name": "Helicone (`helicone`)", "url": "https://docs.litellm.ai/docs/providers/helicone", @@ -5327,6 +5354,32 @@ "generateContent": false } }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "bigmodel": { "display_name": "BigModel (Zhipu AI) (`bigmodel`)", "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 01aa69ba5d2..dd964a74735 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7816,7 +7816,7 @@ "paths": { "/claude-code/marketplace.json": { "get": { - "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout a key the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nWith `general_settings.claude_code_marketplace_auth_required: true` the key must come\nin the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the\nmarketplace source's `headers`), `?key=` is ignored, and a caller with no valid key\ngets a 401 with `WWW-Authenticate: Bearer`.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", "operationId": "get_marketplace_claude_code_marketplace_json_get", "parameters": [ { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index df139a9b84d..4047fa3f0ad 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + claude_code_marketplace_auth_required: bool = Field( + False, + description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.", + ) disable_requester_ip_address_logging: bool | None = Field( None, description="If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.", @@ -3744,6 +3748,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index b0088d93112..fc82686947d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -7,7 +7,8 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on any HTTPS host (S3, Artifactory, a static file server). Endpoints: -/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills) +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (anonymous unless + general_settings.claude_code_marketplace_auth_required; a key adds its granted skills) /claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only) /claude-code/plugins - GET - List plugins visible to the key (enabled, plus granted disabled ones) /claude-code/plugins/{name} - GET - Get plugin details (403 on a disabled plugin the key is not granted) @@ -31,14 +32,17 @@ from pydantic import TypeAdapter from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( SkillVisibility, resolve_skill_visibility, ) +from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository +from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( InstallationPreference, ListPluginsResponse, @@ -109,6 +113,41 @@ async def get_prisma_client() -> object: _get_prisma_client: Final = get_prisma_client +async def _header_key_caller(request: Request) -> UserAPIKeyAuth: + try: + caller: Final = await user_api_key_auth( + request=request, + api_key=request.headers.get(SpecialHeaders.openai_authorization.value) or "", + custom_litellm_key_header=request.headers.get(SpecialHeaders.custom_litellm_api_key.value), + ) + except ProxyException as e: + if e.code != "401": + raise + raise ProxyException( + message=e.message, + type=e.type, + param=e.param, + code=401, + headers={**e.headers, "WWW-Authenticate": "Bearer"}, + ) from e + if caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + raise ProxyException( + message="The LiteLLM key could not be verified because the database is unavailable", + type=ProxyErrorTypes.no_db_connection, + param=None, + code=503, + ) + return caller + + +async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAuth | None: + if marketplace_auth_required(): + return await _header_key_caller(request) + if key: + return await user_api_key_auth(request=request, api_key=f"Bearer {key}") + return None + + @router.get( "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], @@ -121,10 +160,15 @@ async def get_marketplace(request: Request, key: str | None = None): - claude plugin marketplace add - claude plugin install @ - Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + Without a key the catalog holds the enabled (public) plugins. With `?key=sk-...` the key is authenticated and the catalog also holds the disabled plugins granted to it through `object_permission.skills` or an access group on the key or its team. + With `general_settings.claude_code_marketplace_auth_required: true` the key must come + in the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the + marketplace source's `headers`), `?key=` is ignored, and a caller with no valid key + gets a 401 with `WWW-Authenticate: Bearer`. + Returns: Marketplace catalog with list of available plugins and their git sources. @@ -137,10 +181,7 @@ async def get_marketplace(request: Request, key: str | None = None): """ try: prisma_client: Final = await get_prisma_client() - - caller: Final[UserAPIKeyAuth | None] = ( - await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None - ) + caller: Final = await _marketplace_caller(request, key) visibility: Final[SkillVisibility] = await resolve_skill_visibility(caller) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( where=visibility.where() @@ -201,7 +242,7 @@ async def get_marketplace(request: Request, key: str | None = None): verbose_proxy_logger.exception("Error generating marketplace: %s", e) raise HTTPException( status_code=500, - detail={"error": f"Failed to generate marketplace: {e}"}, + detail={"error": "Failed to generate marketplace"}, ) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..1a2e610d5c4 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -2,7 +2,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) """ -from collections.abc import Mapping +from collections.abc import AsyncGenerator, Mapping from types import MappingProxyType from typing import Final @@ -19,12 +19,10 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) -from litellm.llms.base_llm.guardrail_translation.utils import ( - blocked_response_usage as _blocked_response_usage, -) from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( @@ -44,6 +42,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, error_status_code, + litellm_call_id_headers, openai_error_param, openai_error_type, with_litellm_call_id, @@ -58,6 +57,8 @@ from litellm.router import Router router: Final = APIRouter() +_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: if not exc.provider_specific_fields: @@ -229,44 +230,22 @@ async def anthropic_response( request_data=_data, ) - # Create Anthropic-formatted response with violation message - import uuid + block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS + block_handler: Final = AnthropicMessagesHandler() - from litellm.types.utils import AnthropicMessagesResponse + if data.get("stream") is True: + block_chunks: Final = block_handler.build_block_sse_chunks(e) - # Report the blocked LLM response's real token usage (carried on the - # exception) instead of discarding it; zero for pre-call blocks. - _usage: Final = _blocked_response_usage(e.original_response) - - _anthropic_response: Final = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()}", - type="message", - role="assistant", - content=[{"type": "text", "text": e.message}], - model=e.model, - stop_reason="end_turn", - usage=_usage, - ) - - if data.get("stream", None) is not None and data["stream"] is True: - # For streaming, use the standard SSE data generator - async def _passthrough_stream_generator(): - yield _anthropic_response - - selected_data_generator: Final = ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=_passthrough_stream_generator(), - user_api_key_dict=user_api_key_dict, - request_data=_data, - proxy_logging_obj=proxy_logging_obj, - ) + async def _blocked_stream() -> AsyncGenerator[str, None]: + for chunk in block_chunks: + yield chunk.decode() return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers={}, + generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers ) - return _anthropic_response + fastapi_response.headers.update(block_headers) + return block_handler.build_block_response(e) except AnthropicContextManagementError as e: if e.status_code >= 500: # Server-side polyfill failures hit the failure hook for spend/alert diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index 1911cb574ef..6c58f2edc02 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -45,7 +45,11 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_set_request_parsed_body, ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET -from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.management_endpoints.ui_sso import ( + CLI_SSO_BROWSER_TEAM_CHOICE, + CLI_SSO_CHOSEN_TEAM_ID, + CliSsoTeamDetail, +) from litellm.proxy.mcp_server.gateway_dcr_flow import ( PROXY_API_AUDIENCE, MintedProxyCredential, @@ -235,6 +239,7 @@ async def device_authorization(request: Request) -> JSONResponse: "sso_complete": False, "user_code_verified": False, "session_data": None, + CLI_SSO_BROWSER_TEAM_CHOICE: True, } _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) @@ -267,7 +272,15 @@ def _validate_login(flow: Mapping[str, object]) -> _GatewayLogin | _OAuthError: status_code=400, error="invalid_grant", description="The login session is malformed; sign in again" ) - team_id: Final = session_data.teams[0] if session_data.teams else None + chosen: Final = flow.get(CLI_SSO_CHOSEN_TEAM_ID) + chosen_team_id: Final = chosen if isinstance(chosen, str) else None + if chosen_team_id is not None and chosen_team_id not in session_data.teams: + return _OAuthError( + status_code=400, + error="invalid_grant", + description="The team picked at sign-in is no longer one of your teams; sign in again", + ) + team_id: Final = chosen_team_id or next(iter(session_data.teams), None) selected_team: Final = selected_cli_sso_team_detail(team_details=session_data.team_details, team_id=team_id) if selected_team is None: return _OAuthError( diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 098dd2f9816..67728e0fd63 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -35,7 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders -from litellm.utils import resolve_api_key_secret_reference +from litellm.utils import aresolve_api_key_secret_reference if TYPE_CHECKING: from prisma import types as prisma_types @@ -227,7 +227,7 @@ async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_clie return response.status_code == httpx.codes.NOT_FOUND -def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None: +async def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None: model: Final = credentials.get("model") api_base: Final = credentials.get("api_base") api_key: Final = credentials.get("api_key") @@ -244,7 +244,7 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, return None if resolved_api_base is None: return None - return resolved_api_base, resolve_api_key_secret_reference( + return resolved_api_base, await aresolve_api_key_secret_reference( resolved_api_key, model=model, llm_provider=provider, upstream=credentials ) @@ -255,7 +255,7 @@ async def litellm_executed_provider_for( provider: Final = litellm_executed_provider_of(credentials) if provider is None: return None - upstream: Final = _upstream_of(credentials, provider) + upstream: Final = await _upstream_of(credentials, provider) if upstream is None: return None return provider if await lacks_files_api(*upstream) else None diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index fc29a6e3a08..7e487ded569 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -129,6 +129,7 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( + endpoint_unsupported_by_model, route_request, ) from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id @@ -871,6 +872,17 @@ _DECODED_JSON: Final = TypeAdapter(object) _ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +def _fallback_probe_data( + pristine: object, fallback_model: str, key_aliases: object, group_alias: object +) -> Mapping[str, object]: + return { + **_REQUEST_FIELDS.validate_python(pristine), + "model": resolve_served_model( + fallback_model, _DECODED_JSON.validate_python(litellm.model_alias_map), key_aliases, group_alias + ), + } + + def _request_model(data: Mapping[str, object]) -> str | list[str] | None: try: return _REQUEST_MODEL.validate_python(data.get("model"), strict=True) @@ -2676,6 +2688,17 @@ class ProxyBaseLLMRequestProcessing: original_model, fallback_models, ) + raw_router_settings: Final = _DECODED_JSON.validate_python( + _REQUEST_FIELDS.validate_python(rate_limited_data).get("router_settings_override") + ) + group_alias: Final = _DECODED_JSON.validate_python( + _REQUEST_FIELDS.validate_python(raw_router_settings).get("model_group_alias") + if isinstance(raw_router_settings, Mapping) + else None + ) + key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime + object, user_api_key_dict.aliases + ) try: for fallback_model in fallback_models: if fallback_model == original_model: @@ -2691,6 +2714,13 @@ class ProxyBaseLLMRequestProcessing: self.data = independent_snapshot(pristine) self.data["model"] = fallback_model try: + if endpoint_unsupported_by_model( + route_type, + _fallback_probe_data(pristine, fallback_model, key_aliases, group_alias), + llm_router, + user_api_key_dict.team_id, + ): + continue return await self.common_processing_pre_call_logic( request=request, general_settings=general_settings, @@ -4029,13 +4059,13 @@ class ProxyBaseLLMRequestProcessing: (assembled_response, cache_hit); the closure also runs non-apply_guardrail post-call hooks via _run_deferred_stream_guardrails. - - Every other anthropic_messages/aresponses stream gets a closure - that dispatches on the stored args shape, because the arming site - cannot tell the producers apart: native iterators store a single - ready-made logging coroutine to enqueue, while bridged streams - (LiteLLMCompletionStreamingIterator, and the plain SSE generator - AnthropicStreamWrapper returns for bridged /v1/messages) share - their inner CustomStreamWrapper's logging_obj and so store + - Every other anthropic_messages/aresponses/atext_completion stream + gets a closure that dispatches on the stored args shape, because the + arming site cannot tell the producers apart: native iterators store + a single ready-made logging coroutine to enqueue, while bridged + streams (LiteLLMCompletionStreamingIterator, TextCompletionStreamWrapper, + and the plain SSE generator AnthropicStreamWrapper returns for bridged + /v1/messages) share their inner CustomStreamWrapper's logging_obj and so store (assembled_response, cache_hit); for those the closure only dispatches success logging, matching the route's pre-existing hook surface. @@ -4072,7 +4102,11 @@ class ProxyBaseLLMRequestProcessing: logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete return - if route_type not in ("anthropic_messages", "aresponses") or not self._is_streaming_response(response): + if route_type not in ( + "anthropic_messages", + "aresponses", + "atext_completion", + ) or not self._is_streaming_response(response): return from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index f889748c504..d1787f3ff8b 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -436,10 +436,7 @@ def get_model_group_from_litellm_kwargs(kwargs: dict) -> str | None: def get_model_group_from_request_data(data: dict) -> str | None: _metadata: Final = data.get("metadata", None) or {} _model_group: Final = _metadata.get("model_group", None) - if _model_group is not None: - return _model_group - - return None + return _model_group if isinstance(_model_group, str) else None def get_remaining_tokens_and_requests_from_request_data(data: dict) -> dict[str, str]: diff --git a/litellm/proxy/common_utils/claude_code_marketplace_auth.py b/litellm/proxy/common_utils/claude_code_marketplace_auth.py new file mode 100644 index 00000000000..0cfb3061744 --- /dev/null +++ b/litellm/proxy/common_utils/claude_code_marketplace_auth.py @@ -0,0 +1,17 @@ +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +_AUTH_REQUIRED_SETTING: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None) + + +def marketplace_auth_required() -> bool: + from litellm.proxy.proxy_server import general_settings_view + + try: + return ( + _AUTH_REQUIRED_SETTING.validate_python(general_settings_view().get("claude_code_marketplace_auth_required")) + is True + ) + except ValidationError: + return True diff --git a/litellm/proxy/common_utils/html_forms/native_client_consent.py b/litellm/proxy/common_utils/html_forms/native_client_consent.py index dac92c4e787..aeadc93221c 100644 --- a/litellm/proxy/common_utils/html_forms/native_client_consent.py +++ b/litellm/proxy/common_utils/html_forms/native_client_consent.py @@ -62,7 +62,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p

Approving issues it a personal credential that expires within {CLI_JWT_EXPIRATION_HOURS} hours. lite logout stops it from being renewed. Only approve if you started this sign-in yourself.

-{_team_field(teams)} +{team_select_field(teams)}
@@ -74,7 +74,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p """ -def _team_field(teams: Sequence[tuple[str, str]]) -> str: +def team_select_field(teams: Sequence[tuple[str, str]], *, require_choice: bool = False) -> str: if not teams: return "" if len(teams) == 1: @@ -83,9 +83,12 @@ def _team_field(teams: Sequence[tuple[str, str]]) -> str: f'' f"

Requests are attributed to team {escape(team_label)}.

" ) + placeholder: Final = '' if require_choice else "" options: Final = "".join( f'' for team_id, team_label in teams ) + required: Final = " required" if require_choice else "" return ( - f'' + '' + f'' ) diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 83cc4501466..d126f112f90 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -4,7 +4,7 @@ import os import subprocess from typing import Final -from litellm._logging import print_startup_message, verbose_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_logger def extract_sql_commands(diff_output: str) -> list[str]: @@ -63,7 +63,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: run_prisma, ) except ImportError as e: - print_startup_message( + print_startup_warning( f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}", ) return False, [] @@ -96,14 +96,14 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: else: return False, [] except subprocess.TimeoutExpired: - print_startup_message( + print_startup_warning( f"Timed out after {timeout}s generating the migration diff. " f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer.", ) return False, [] except subprocess.CalledProcessError as e: error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}" - print_startup_message(error_message) + print_startup_warning(error_message) return False, [] diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 8ac2597c99b..16dbe2b4131 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1900,8 +1900,6 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, ) - # NOTE: Daily tag spend is committed by a separate scheduler job. - ################## Daily End-User Spend Update Transactions ################## # Aggregate all in memory daily end-user spend transactions and commit to db await self._flush_daily_spend_queue( diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 7f5efcdaec3..170eb450339 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -421,7 +421,10 @@ class SpendLogCleanup: general_settings=None, redis_cache: RedisCache | None = None, partition_manager: SpendLogsPartitionManager | None = None, + *, + clock: Callable[[], float] = time.monotonic, ): + self._clock = clock self.retention_seconds: int | None = None self.partition_manager = partition_manager or SpendLogsPartitionManager() from litellm.proxy.proxy_server import general_settings as default_settings @@ -586,7 +589,7 @@ class SpendLogCleanup: async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None: """Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -624,7 +627,7 @@ class SpendLogCleanup: stale.allocated_bytes, _ANALYZE_SQL, ) - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -653,7 +656,7 @@ class SpendLogCleanup: async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None: """The reclaimable count saved by the last check, or None on the first check or when it cannot be read.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -686,7 +689,7 @@ class SpendLogCleanup: the tool-index rows that expire with their spend logs. When fewer rows than that remain, the step is everything up to now. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) async with db_span("size_step_cutoff", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx: @@ -718,11 +721,11 @@ class SpendLogCleanup: Interpolating this into SQL is safe by construction: an int cannot carry SQL, and SET does not accept a bind parameter. """ - remaining_ms: Final = int((deadline - time.monotonic()) * 1000) + remaining_ms: Final = int((deadline - self._clock()) * 1000) return max(1, min(int(self.batch_timeout_seconds * 1000), remaining_ms)) @staticmethod - def _group_deadline(overall_deadline: float, groups_remaining: int) -> float: + def _group_deadline(overall_deadline: float, groups_remaining: int, now: float) -> float: """ Give each pending cleanup group an equal share of the time left. @@ -731,10 +734,9 @@ class SpendLogCleanup: """ if groups_remaining == 1: return overall_deadline - current_time: Final = time.monotonic() - if current_time >= overall_deadline: + if now >= overall_deadline: return overall_deadline - return current_time + (overall_deadline - current_time) / groups_remaining + return now + (overall_deadline - now) / groups_remaining def _remaining_timeout_ms(self, deadline: float) -> RemainingTimeoutMs: """ @@ -748,7 +750,7 @@ class SpendLogCleanup: """ def remaining() -> int | None: - return None if time.monotonic() >= deadline else self._timeout_ms(deadline) + return None if self._clock() >= deadline else self._timeout_ms(deadline) return remaining @@ -841,7 +843,7 @@ class SpendLogCleanup: run_count = 0 consecutive_failures = 0 while True: - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info( "Run budget exhausted during %s cleanup after %d rows; the next run resumes from here", table_name, @@ -858,13 +860,13 @@ class SpendLogCleanup: prisma_client, cutoff_date, table_name, time_column, total_deleted, "batch_cap_reached", deadline ) # Find rows and delete them in one go without fetching to application - batch_started_at = time.monotonic() + batch_started_at = self._clock() try: batch_result = await self._execute_delete_batch( prisma_client, delete_sql, cutoff_date, table_name, deadline ) except Exception as batch_exc: - if time.monotonic() >= deadline: + if self._clock() >= deadline: # The statement timeout was clamped to the budget that was # left, so this batch was cancelled by the deadline itself. # That is the bound working, not a database fault, and @@ -920,7 +922,7 @@ class SpendLogCleanup: consecutive_failures = 0 deleted_count = batch_result - SpendLogCleanupMetrics.record_batch(table_name, deleted_count, time.monotonic() - batch_started_at) + SpendLogCleanupMetrics.record_batch(table_name, deleted_count, self._clock() - batch_started_at) verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name) if deleted_count == 0: @@ -957,7 +959,7 @@ class SpendLogCleanup: operator the backlog was not drained; the gauge simply keeps its value from the last run that finished inside its budget. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return TableCleanupResult(table_name=table_name, rows_deleted=rows_deleted, stop_reason=stop_reason) remaining: Final = await self._count_remaining(prisma_client, cutoff_date, table_name, time_column, deadline) if remaining is not None: @@ -1070,7 +1072,7 @@ class SpendLogCleanup: # behind any long-running reader for as long as that reader lives, which # is the one way this job could still outlast its budget without bound. remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info("Run budget already spent, skipping partition maintenance this run") elif self.general_settings.get( "use_spend_logs_partitioning", False @@ -1082,7 +1084,7 @@ class SpendLogCleanup: verbose_proxy_logger.info("Dropped %d expired spend-log partitions: %s", len(dropped), dropped) logs_result: Final = await self._delete_old_logs( - prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2) + prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2, now=self._clock()) ) verbose_proxy_logger.info("Deleted %s logs", logs_result.rows_deleted) @@ -1210,7 +1212,7 @@ class SpendLogCleanup: self, prisma_client: PrismaClient, deadline: float, still_over_target: Callable[[], Awaitable[bool]] ) -> tuple[str, ...]: remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline or not self._setting("use_spend_logs_partitioning"): + if self._clock() >= deadline or not self._setting("use_spend_logs_partitioning"): return () if not await self.partition_manager.is_partitioned(prisma_client, remaining_timeout_ms): return () @@ -1257,7 +1259,7 @@ class SpendLogCleanup: needed: Final = rows_still_needed(state.logs_deleted + state.index_deleted) if needed <= 0: return state.stopped("target_reached") - if time.monotonic() >= deadline: + if self._clock() >= deadline: return state.stopped("budget_exhausted") if state.batches_left <= 0: return state.stopped("batch_cap_reached") @@ -1318,11 +1320,11 @@ class SpendLogCleanup: except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job verbose_proxy_logger.warning("Auto-router baseline retention remains pending") sessions_result: Final = await self._delete_old_autorouter_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 3) + prisma_client, session_cutoff, self._group_deadline(deadline, 3, self._clock()) ) verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted) user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 2) + prisma_client, session_cutoff, self._group_deadline(deadline, 2, self._clock()) ) verbose_proxy_logger.info( "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted @@ -1443,7 +1445,7 @@ class SpendLogCleanup: async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None: lock_acquired = False - run_started_at: Final = time.monotonic() + run_started_at: Final = self._clock() progress: Final = _RunProgress() progress_token: Final = _run_progress.set(progress) try: @@ -1493,7 +1495,7 @@ class SpendLogCleanup: SpendLogCleanupMetrics.record_run("skipped_locked") return - deadline: Final = time.monotonic() + self.run_budget_seconds + deadline: Final = self._clock() + self.run_budget_seconds size_group: Final = int(active_size_rule is not None) configured_group_count: Final = ( int(delete_spend_logs and self.retention_seconds is not None) @@ -1506,7 +1508,7 @@ class SpendLogCleanup: spend_log_results: Final = ( await self._clean_spend_log_tables( prisma_client, - self._group_deadline(deadline, configured_group_count), + self._group_deadline(deadline, configured_group_count, self._clock()), ) if delete_spend_logs and self.retention_seconds is not None else () @@ -1526,7 +1528,7 @@ class SpendLogCleanup: await self._clean_session_rollup( prisma_client, autorouter_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_spend_logs), + self._group_deadline(deadline, remaining_groups_after_spend_logs, self._clock()), ) if autorouter_retention_seconds is not None else () @@ -1535,7 +1537,7 @@ class SpendLogCleanup: await self._clean_health_checks( prisma_client, health_check_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_sessions), + self._group_deadline(deadline, remaining_groups_after_sessions, self._clock()), ) if health_check_retention_seconds is not None else () @@ -1544,7 +1546,7 @@ class SpendLogCleanup: await self._clean_daily_tag_spend( prisma_client, daily_tag_spend_retention_seconds, - self._group_deadline(deadline, 1 + size_group), + self._group_deadline(deadline, 1 + size_group, self._clock()), ) if daily_tag_spend_retention_seconds is not None else () @@ -1561,12 +1563,12 @@ class SpendLogCleanup: ) outcome: Final = self._run_outcome(results) SpendLogCleanupMetrics.record_run(outcome) - self._log_run_summary(outcome, results, time.monotonic() - run_started_at, size_report) + self._log_run_summary(outcome, results, self._clock() - run_started_at, size_report) except asyncio.CancelledError: verbose_proxy_logger.error( "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here", - time.monotonic() - run_started_at, + self._clock() - run_started_at, progress.rows_deleted, progress.batches, ) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 3d783d8aba1..be5de040fdf 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -1127,16 +1127,16 @@ class PrismaManager: return ProxyExtrasDBManager.build_request_log_indexes() @staticmethod - def migration_data_check_report(use_migrate: bool = True) -> str | None: - """The read-only pre-migration row check on its own, for `--check_migrations_against_existing_data`: the - refusal the migration step would print, or None when the pending schema changes - can be applied to the rows the database has.""" + def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]: + """`--check_migrations_against_existing_data` on its own: the pending migrations applied inside a + transaction that is rolled back, as (whether they all applied, what the operator reads).""" from litellm_proxy_extras.utils import ( ProxyExtrasDBManager, # pyright: ignore[reportUnknownVariableType] # litellm-proxy-extras is an optional install ) + from pydantic import TypeAdapter - refusal: Final[object] = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm-proxy-extras is an optional install - return refusal if isinstance(refusal, str) else None + summary: Final[object] = ProxyExtrasDBManager.migration_dry_run_summary(use_migrate=use_migrate) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm-proxy-extras is an optional install + return TypeAdapter(tuple[bool, str]).validate_python(summary) @staticmethod def start_request_log_index_build() -> None: diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 792ef8a3f8c..36f424507d7 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.html_forms.default_credentials_hint import should_hide_default_credentials_hint from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_enabled from litellm.types.proxy.discovery_endpoints.ui_discovery_endpoints import ( @@ -51,4 +52,5 @@ async def get_ui_config(): workers=proxy_config.worker_registry if is_control_plane else [], mcp_stdio_enabled=is_mcp_stdio_enabled(), use_si_terminology=use_si_terminology, + claude_code_marketplace_auth_required=marketplace_auth_required(), ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index 7ff5aa2466b..e57d6e78378 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -82,6 +82,22 @@ def _as_plain_dict(item: object) -> Mapping[str, object]: return cast("Mapping[str, object]", item) # cast-ok: wire rows are message/tool-call dicts +_STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +def _str_keyed(value: object) -> Mapping[str, object]: + return _STR_KEYED_MAPPING.validate_python(value) if isinstance(value, Mapping) else {} + + +def _request_route(request_data: Mapping[str, object]) -> str | None: + litellm_metadata: Final = request_data.get("litellm_metadata") + metadata_bucket: Final = _str_keyed( + litellm_metadata if litellm_metadata is not None else request_data.get("metadata") + ) + route: Final = metadata_bucket.get("user_api_key_request_route") + return route if isinstance(route, str) else None + + class GraySwanGuardrailMissingSecrets(Exception): """Raised when the Gray Swan API key is missing.""" @@ -625,11 +641,8 @@ class GraySwanGuardrail(CustomGuardrail): from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.llms import load_guardrail_translation_mappings - litellm_metadata: Final = request_data.get("litellm_metadata") - request_route: Final = ( - litellm_metadata.get("user_api_key_request_route") if isinstance(litellm_metadata, Mapping) else None - ) - route_call_types: Final = get_call_types_for_route(request_route) if isinstance(request_route, str) else None + request_route: Final = _request_route(request_data) + route_call_types: Final = get_call_types_for_route(request_route) if request_route is not None else None call_type: Final = ( (route_call_types[0].value if route_call_types else None) or (logging_obj.call_type if logging_obj is not None else None) @@ -698,13 +711,12 @@ class GraySwanGuardrail(CustomGuardrail): inbound_headers: Final = self._extract_inbound_headers(request_data, logging_obj) - litellm_metadata: Final = request_data.get("litellm_metadata") - cleaned_litellm_metadata: Final = dict(litellm_metadata) if isinstance(litellm_metadata, dict) else {} - if inbound_headers: - existing_headers: Final = cleaned_litellm_metadata.get("headers") - cleaned_litellm_metadata["headers"] = ( - {**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers - ) + litellm_metadata: Final = _str_keyed(request_data.get("litellm_metadata")) + cleaned_litellm_metadata: Final = ( + {**litellm_metadata, "headers": {**_str_keyed(litellm_metadata.get("headers")), **inbound_headers}} + if inbound_headers + else litellm_metadata + ) if cleaned_litellm_metadata: sanitized: Final = _DECODED_JSON.validate_python( safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index f58c8705815..71b8976bebf 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -12,11 +12,11 @@ import json import os import re import time -from collections.abc import AsyncGenerator, Coroutine, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass, replace from datetime import datetime from re import Pattern -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, cast import yaml from fastapi import HTTPException @@ -40,6 +40,7 @@ from litellm.proxy.guardrails.content_filter_data import ( ) from litellm.types.utils import ( CallTypes, + Delta, Function, GenericGuardrailAPIInputs, GuardrailStatus, @@ -47,6 +48,9 @@ from litellm.types.utils import ( Message, ModelResponse, ModelResponseStream, + StreamingChoices, + TextChoices, + TextCompletionResponse, ) if TYPE_CHECKING: @@ -78,6 +82,7 @@ from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 _DECODED_ARGUMENTS: Final = TypeAdapter(object) +_StreamItemT: Final = TypeVar("_StreamItemT") _JSON_ENTRIES: Final = TypeAdapter(list[object]) GAP_WORD_TOKENIZER: Final = re.compile(r"\b\w+\b") SENTENCE_TERMINATORS: Final = re.compile(r"[.!?]+") @@ -139,6 +144,26 @@ class _CategoryConfigView(TypedDict): category_file: str | None +def _streamed_choice_content(choice: StreamingChoices | TextChoices) -> str | None: + if isinstance(choice, TextChoices): + return choice.text if isinstance(choice.text, str) else "" + delta: Final = getattr(choice, "delta", None) + if not delta: + return None + content: Final = getattr(delta, "content", None) + return content if isinstance(content, str) else "" + + +def _streamed_choices(item: object) -> Sequence[StreamingChoices | TextChoices]: + if isinstance(item, (ModelResponseStream, TextCompletionResponse)): + return item.choices + return () + + +def _is_choiceless_chunk(item: object) -> bool: + return isinstance(item, (ModelResponseStream, TextCompletionResponse)) and not item.choices + + @dataclass(frozen=True, slots=True) class _StreamedChoiceState: buffered_text: str = "" @@ -146,6 +171,52 @@ class _StreamedChoiceState: committed_detections: tuple[ContentFilterDetection, ...] = () latest_detections: tuple[ContentFilterDetection, ...] = () next_trim_len: int = 0 + finished: bool = False + + +def _end_of_stream_chunk( + last_item: object, state_by_choice: Mapping[int, _StreamedChoiceState] +) -> ModelResponseStream | TextCompletionResponse | None: + unfinished: Final = tuple(index for index, state in state_by_choice.items() if not state.finished) + if not unfinished: + return None + if isinstance(last_item, TextCompletionResponse): + return TextCompletionResponse( + id=last_item.id, + created=last_item.created, + model=last_item.model, + object="text_completion", + choices=[TextChoices(index=index, text="") for index in unfinished], + ).model_copy(update={"usage": None}) + if isinstance(last_item, ModelResponseStream): + return ModelResponseStream( + id=last_item.id, + created=last_item.created, + model=last_item.model, + choices=[StreamingChoices(index=index, delta=Delta(content="")) for index in unfinished], + ) + return None + + +async def _stream_then_end_of_stream_chunk( + response: AsyncIterable[_StreamItemT], end_of_stream_chunk: Callable[[_StreamItemT], _StreamItemT | None] +) -> AsyncGenerator[tuple[_StreamItemT, bool], None]: + last_item: _StreamItemT | None = None # rebind-ok: the end-of-stream chunk copies the last chunk's id and model + held_items: tuple[_StreamItemT, ...] = () # rebind-ok: the usage chunk must stay after the end-of-stream chunk + async for item in response: + last_item = item + if _is_choiceless_chunk(item): + held_items = (*held_items, item) + continue + for held_item in held_items: + yield held_item, False + held_items = () + yield item, False + closing_item: Final = None if last_item is None else end_of_stream_chunk(last_item) + if closing_item is not None: + yield closing_item, True + for held_item in held_items: + yield held_item, False @dataclass(frozen=True, slots=True) @@ -2104,72 +2175,75 @@ class ContentFilterGuardrail(CustomGuardrail): ) try: - async for item in response: - if isinstance(item, ModelResponseStream) and item.choices: - for choice in item.choices: - if not (hasattr(choice, "delta") and choice.delta): - continue + async for item, stream_ended in _stream_then_end_of_stream_chunk( + response, lambda last_item: _end_of_stream_chunk(last_item, state_by_choice) + ): + for choice in _streamed_choices(item): + new_content = _streamed_choice_content(choice) + if new_content is None: + continue - choice_index = getattr(choice, "index", 0) - if not isinstance(choice_index, int): - choice_index = 0 + choice_index = getattr(choice, "index", 0) + if not isinstance(choice_index, int): + choice_index = 0 - content = getattr(choice.delta, "content", None) - is_final = bool(getattr(choice, "finish_reason", None)) - new_content = content if isinstance(content, str) else "" - if not new_content and not is_final: - continue + is_final = stream_ended or bool(getattr(choice, "finish_reason", None)) + if not new_content and not is_final: + continue - previous_state = state_by_choice.get(choice_index, _StreamedChoiceState()) - buffered_text = previous_state.buffered_text + new_content - if not buffered_text: - continue + previous_state = state_by_choice.get(choice_index, _StreamedChoiceState()) + buffered_text = previous_state.buffered_text + new_content + if not buffered_text: + continue - # Add a space at the end if it's the final chunk to trigger word boundaries (\b) - text_to_scan = buffered_text + (" " if is_final else "") - choice_detections: list[ContentFilterDetection] = [] - scan_started = time.perf_counter() + # Add a space at the end if it's the final chunk to trigger word boundaries (\b) + text_to_scan = buffered_text + (" " if is_final else "") + choice_detections: list[ContentFilterDetection] = [] + scan_started = time.perf_counter() - try: - masked_text = self._filter_single_text(text_to_scan, detections=choice_detections) - if is_final and masked_text.endswith(" "): - masked_text = masked_text[:-1] - latest_detections = tuple(choice_detections) - except HTTPException: - state_by_choice[choice_index] = replace( - previous_state, latest_detections=tuple(choice_detections) - ) - raise - except Exception as e: - verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e) - masked_text = text_to_scan # Fallback to current text - latest_detections = previous_state.latest_detections - finally: - scan_seconds += time.perf_counter() - scan_started - - safe_to_yield_len = max( - previous_state.yielded_masked_text_len, - len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS), + try: + masked_text = self._filter_single_text(text_to_scan, detections=choice_detections) + if is_final and masked_text.endswith(" "): + masked_text = masked_text[:-1] + latest_detections = tuple(choice_detections) + except HTTPException: + state_by_choice[choice_index] = replace( + previous_state, latest_detections=tuple(choice_detections) ) - choice.delta.content = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len] - next_state = replace( - previous_state, - buffered_text=buffered_text, - yielded_masked_text_len=safe_to_yield_len, - latest_detections=latest_detections, - ) - if is_final: - state_by_choice[choice_index] = next_state - continue + raise + except Exception as e: + verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e) + masked_text = text_to_scan # Fallback to current text + latest_detections = previous_state.latest_detections + finally: + scan_seconds += time.perf_counter() - scan_started - trim_started = time.perf_counter() - state_by_choice[choice_index] = self._trim_streamed_choice_buffer(next_state, masked_text, plan) - scan_seconds += time.perf_counter() - trim_started + safe_to_yield_len = max( + previous_state.yielded_masked_text_len, + len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS), + ) + released_text = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len] + if isinstance(choice, TextChoices): + choice.text = released_text + else: + choice.delta.content = released_text + next_state = replace( + previous_state, + buffered_text=buffered_text, + yielded_masked_text_len=safe_to_yield_len, + latest_detections=latest_detections, + ) + if is_final: + state_by_choice[choice_index] = replace(next_state, finished=True) + continue - yield item - else: - # Not a ModelResponseStream or no choices - yield as is - yield item + trim_started = time.perf_counter() + state_by_choice[choice_index] = self._trim_streamed_choice_buffer(next_state, masked_text, plan) + scan_seconds += time.perf_counter() - trim_started + + if stream_ended and not any(_streamed_choice_content(choice) for choice in _streamed_choices(item)): + continue + yield item except HTTPException: status = "guardrail_intervened" raise diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 0f66135d038..8e2fea415ef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -647,9 +647,10 @@ class PromptSecurityGuardrail(CustomGuardrail): async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict: """Process and sanitize document/file items.""" doc_data: Final = item.get("document") or item.get("file") or item + url_key: Final = "file_data" if isinstance(doc_data, dict) and not doc_data.get("url") else "url" if isinstance(doc_data, dict): - url = doc_data.get("url", "") + url = doc_data.get(url_key) or "" doc_content = doc_data.get("data", "") else: url = doc_data if isinstance(doc_data, str) else "" @@ -698,7 +699,7 @@ class PromptSecurityGuardrail(CustomGuardrail): if url.startswith("data:") and header: sanitized_url: Final = f"{header},{sanitized_encoded}" if isinstance(doc_data, dict): - doc_data["url"] = sanitized_url + doc_data[url_key] = sanitized_url elif isinstance(doc_data, dict): doc_data["data"] = sanitized_encoded diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 62c973196b9..d689ab2a537 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -2,7 +2,10 @@ import asyncio import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final + +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -19,6 +22,17 @@ if TYPE_CHECKING: from litellm.router import Router +@with_config(ConfigDict(extra="allow", strict=True)) +class _CachedHealthCheckResults(TypedDict): + healthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + unhealthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + timestamp: NotRequired[ReadOnly[int | float]] + checked_by: NotRequired[ReadOnly[object]] + + +_CACHED_HEALTH_CHECK_RESULTS: Final = TypeAdapter(_CachedHealthCheckResults) + + class SharedHealthCheckManager: """ Manager for coordinating health checks across multiple pods using Redis. @@ -109,7 +123,7 @@ class SharedHealthCheckManager: verbose_proxy_logger.error("Error releasing health check lock: %s", str(e)) @with_service_target(HEALTH_CHECKS_TARGET) - async def get_cached_health_check_results(self) -> dict[str, Any] | None: + async def get_cached_health_check_results(self) -> _CachedHealthCheckResults | None: """ Get cached health check results from Redis. @@ -127,10 +141,9 @@ class SharedHealthCheckManager: return None # Parse the cached data - if isinstance(cached_data, str): - cached_results = json.loads(cached_data) - else: - cached_results = cached_data + cached_results: Final = _CACHED_HEALTH_CHECK_RESULTS.validate_python( + json.loads(cached_data) if isinstance(cached_data, str) else cached_data + ) # Check if the cache is still valid cache_timestamp: Final = cached_results.get("timestamp", 0) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 3dceab6b7f7..b79154bf509 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -865,7 +865,7 @@ async def _recorded_autorouter_savings( entity_id_field="user_id", entity_id=user_id, exclude_entity_ids=None, - api_key=api_key, + api_key=(api_key,) if api_key is not None else None, start_date=start_day, end_date=end_day, model=None, diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 66f386b792d..98372b64015 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -23,6 +23,12 @@ from litellm.proxy.spend_tracking.key_metadata_recovery import ( ) from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled from litellm.proxy.utils import PrismaClient +from litellm.repositories.daily_activity_buckets import ( + api_key_bucket, + model_bucket, + model_group_bucket, + raw_api_key_filter, +) from litellm.repositories.daily_activity_repository import DailyActivityRepository from litellm.types.proxy.management_endpoints.common_daily_activity import ( BreakdownMetrics, @@ -182,7 +188,9 @@ class _AggregatedSpendData(TypedDict): totals: ReadOnly[SpendMetrics] -def _key_metadata(api_key_metadata: Mapping[str, KeyMetadataRow], api_key: str) -> KeyMetadata: +def _key_metadata(api_key_metadata: Mapping[str, KeyMetadataRow], api_key: str | None) -> KeyMetadata: + if not api_key: + return KeyMetadata(key_alias="Unattributed", key_exists=False) meta: Final = api_key_metadata.get(api_key) return KeyMetadata( key_alias=meta.key_alias if meta is not None else None, @@ -302,13 +310,14 @@ def update_breakdown_metrics( parent bucket but never appear as an api_key row, and are kept out of the per-request provider breakdown.""" + api_key: Final = api_key_bucket(record.api_key) is_ptu_sentinel: Final = record.api_key == PTU_SENTINEL_API_KEY # A PTU sentinel row keys on the deployment id so a rename cannot move it, and carries # the operator-facing name in model_group. The breakdown key is rendered directly as a # label, so display the name; two deployments sharing one name merge here, which is # what the write path used to do by collapsing them into a single row. - model_key: Final = (record.model_group or record.model) if is_ptu_sentinel else record.model + model_key: Final = (record.model_group or record.model) if is_ptu_sentinel else model_bucket(record.model) # Update model breakdown if model_key and model_key not in breakdown.models: @@ -321,18 +330,18 @@ def update_breakdown_metrics( if not is_ptu_sentinel: # Update API key breakdown for this model - if record.api_key not in breakdown.models[model_key].api_key_breakdown: - breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( + if api_key not in breakdown.models[model_key].api_key_breakdown: + breakdown.models[model_key].api_key_breakdown[api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), metadata=_key_metadata(api_key_metadata, record.api_key), ) - breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics( - breakdown.models[model_key].api_key_breakdown[record.api_key].metrics, + breakdown.models[model_key].api_key_breakdown[api_key].metrics = update_metrics( + breakdown.models[model_key].api_key_breakdown[api_key].metrics, record, ) # Update model group breakdown - model_group_key: Final = record.model_group or record.model + model_group_key: Final = model_group_bucket(record.model_group, record.model) if model_group_key and model_group_key not in breakdown.model_groups: breakdown.model_groups[model_group_key] = MetricWithMetadata( metrics=SpendMetrics(), @@ -345,13 +354,13 @@ def update_breakdown_metrics( if not is_ptu_sentinel: # Update API key breakdown for this model - if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown: - breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( + if api_key not in breakdown.model_groups[model_group_key].api_key_breakdown: + breakdown.model_groups[model_group_key].api_key_breakdown[api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), metadata=_key_metadata(api_key_metadata, record.api_key), ) - breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics( - breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics, + breakdown.model_groups[model_group_key].api_key_breakdown[api_key].metrics = update_metrics( + breakdown.model_groups[model_group_key].api_key_breakdown[api_key].metrics, record, ) @@ -365,21 +374,20 @@ def update_breakdown_metrics( breakdown.mcp_servers[record.mcp_namespaced_tool_name].metrics, record ) - # Update API key breakdown for this MCP server - if record.api_key not in breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown: - breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = ( - KeyMetricWithMetadata( - metrics=SpendMetrics(), - metadata=_key_metadata(api_key_metadata, record.api_key), + if not is_ptu_sentinel: + # Update API key breakdown for this MCP server + if api_key not in breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown: + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[api_key] = ( + KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=_key_metadata(api_key_metadata, record.api_key), + ) ) - ) - breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ - record.api_key - ].metrics = update_metrics( - breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key].metrics, - record, - ) + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[api_key].metrics = update_metrics( + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[api_key].metrics, + record, + ) if not is_ptu_sentinel: # Update provider breakdown @@ -392,13 +400,13 @@ def update_breakdown_metrics( breakdown.providers[provider].metrics = update_metrics(breakdown.providers[provider].metrics, record) # Update API key breakdown for this provider - if record.api_key not in breakdown.providers[provider].api_key_breakdown: - breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( + if api_key not in breakdown.providers[provider].api_key_breakdown: + breakdown.providers[provider].api_key_breakdown[api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), metadata=_key_metadata(api_key_metadata, record.api_key), ) - breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics( - breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, + breakdown.providers[provider].api_key_breakdown[api_key].metrics = update_metrics( + breakdown.providers[provider].api_key_breakdown[api_key].metrics, record, ) @@ -413,25 +421,26 @@ def update_breakdown_metrics( breakdown.endpoints[record.endpoint].metrics, record ) - # Update API key breakdown for this endpoint - if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown: - breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( - metrics=SpendMetrics(), - metadata=_key_metadata(api_key_metadata, record.api_key), + if not is_ptu_sentinel: + # Update API key breakdown for this endpoint + if api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown: + breakdown.endpoints[record.endpoint].api_key_breakdown[api_key] = KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=_key_metadata(api_key_metadata, record.api_key), + ) + breakdown.endpoints[record.endpoint].api_key_breakdown[api_key].metrics = update_metrics( + breakdown.endpoints[record.endpoint].api_key_breakdown[api_key].metrics, + record, ) - breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics( - breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics, - record, - ) if not is_ptu_sentinel: # Update api key breakdown - if record.api_key not in breakdown.api_keys: - breakdown.api_keys[record.api_key] = KeyMetricWithMetadata( + if api_key not in breakdown.api_keys: + breakdown.api_keys[api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), metadata=_key_metadata(api_key_metadata, record.api_key), ) - breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record) + breakdown.api_keys[api_key].metrics = update_metrics(breakdown.api_keys[api_key].metrics, record) # Update entity-specific metrics if entity_id_field is provided if entity_id_field: @@ -445,13 +454,13 @@ def update_breakdown_metrics( if not is_ptu_sentinel: # Update API key breakdown for this entity - if record.api_key not in breakdown.entities[entity_value].api_key_breakdown: - breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata( + if api_key not in breakdown.entities[entity_value].api_key_breakdown: + breakdown.entities[entity_value].api_key_breakdown[api_key] = KeyMetricWithMetadata( metrics=SpendMetrics(), metadata=_key_metadata(api_key_metadata, record.api_key), ) - breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics( - breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics, + breakdown.entities[entity_value].api_key_breakdown[api_key].metrics = update_metrics( + breakdown.entities[entity_value].api_key_breakdown[api_key].metrics, record, ) @@ -566,7 +575,7 @@ def daily_activity_scope( entity_id_field: str, entity_id: str | list[str] | None, exclude_entity_ids: list[str] | None, - api_key: str | list[str] | None, + api_key: str | Sequence[str] | None, start_date: str, end_date: str, model: str | None, @@ -577,9 +586,7 @@ def daily_activity_scope( entity_ids: tuple[str, ...] | None = ( (entity_id,) if isinstance(entity_id, str) else tuple(entity_id) if entity_id is not None else None ) - api_keys: tuple[str, ...] | None = ( - None if api_key in (None, "") else (api_key,) if isinstance(api_key, str) else tuple(api_key) - ) + api_keys: Final = raw_api_key_filter(api_key) return DailyActivityScope( table=table_value, entity_id_field=entity_id_field, @@ -715,6 +722,29 @@ _GROUP_DATE_ENDPOINT: Final = 62 # 0b0111110 _GROUP_DATE_ENDPOINT_API_KEY: Final = 30 # 0b0011110 +def sum_spend_metrics(metrics: Sequence[SpendMetrics]) -> SpendMetrics: + return SpendMetrics( + spend=sum(metric.spend for metric in metrics), + flat_cost=sum(metric.flat_cost for metric in metrics), + ptu_hours=sum(metric.ptu_hours for metric in metrics), + prompt_tokens=sum(metric.prompt_tokens for metric in metrics), + completion_tokens=sum(metric.completion_tokens for metric in metrics), + cache_read_input_tokens=sum(metric.cache_read_input_tokens for metric in metrics), + cache_creation_input_tokens=sum(metric.cache_creation_input_tokens for metric in metrics), + compression_saved_tokens=sum(metric.compression_saved_tokens for metric in metrics), + compression_savings_spend=sum(metric.compression_savings_spend for metric in metrics), + prompt_caching_savings_spend=sum(metric.prompt_caching_savings_spend for metric in metrics), + gateway_injected_caching_savings_spend=sum(metric.gateway_injected_caching_savings_spend for metric in metrics), + autorouter_savings_spend=sum(metric.autorouter_savings_spend for metric in metrics), + total_tokens=sum(metric.total_tokens for metric in metrics), + successful_requests=sum(metric.successful_requests for metric in metrics), + failed_requests=sum(metric.failed_requests for metric in metrics), + api_requests=sum(metric.api_requests for metric in metrics), + total_response_time_ms=sum(metric.total_response_time_ms for metric in metrics), + timed_requests=sum(metric.timed_requests for metric in metrics), + ) + + def _record_to_spend_metrics(record: RollupMetricsRow) -> SpendMetrics: """Build a SpendMetrics directly from one already-aggregated rollup row. @@ -753,8 +783,7 @@ def _aggregate_grouping_sets_records_sync( Each row carries a `group_level` bitmask (from Postgres GROUPING()) that identifies which rollup level it belongs to. We dispatch the row's - pre-aggregated metrics straight into the matching bucket — no per-row - summing in Python and no nested update_metrics calls. + pre-aggregated metrics into the matching bucket, combining normalized names. """ total_metrics = SpendMetrics() grouped_data: Final[dict[str, GroupedData]] = {} @@ -771,20 +800,23 @@ def _aggregate_grouping_sets_records_sync( if existing is None: target[key] = MetricWithMetadata(metrics=metrics, metadata={}) else: - existing.metrics = metrics + existing.metrics = sum_spend_metrics((existing.metrics, metrics)) def assign_api_key_breakdown( target: dict[str, MetricWithMetadata], parent_key: str, - api_key: str, + api_key: str | None, metrics: SpendMetrics, ) -> None: parent = target.get(parent_key) if parent is None: parent = MetricWithMetadata(metrics=SpendMetrics(), metadata={}) target[parent_key] = parent - parent.api_key_breakdown[api_key] = KeyMetricWithMetadata( - metrics=metrics, metadata=_key_metadata(api_key_metadata, api_key) + bucket_key: Final = api_key_bucket(api_key) + previous: Final = parent.api_key_breakdown.get(bucket_key) + parent.api_key_breakdown[bucket_key] = KeyMetricWithMetadata( + metrics=sum_spend_metrics((previous.metrics, metrics)) if previous is not None else metrics, + metadata=_key_metadata(api_key_metadata, api_key), ) for record in records: @@ -803,25 +835,23 @@ def _aggregate_grouping_sets_records_sync( breakdown = ensure_date(record.date)["breakdown"] if level == _GROUP_DATE_API_KEY: - if record.api_key and not is_ptu_sentinel: - breakdown.api_keys[record.api_key] = KeyMetricWithMetadata( + if not is_ptu_sentinel: + breakdown.api_keys[api_key_bucket(record.api_key)] = KeyMetricWithMetadata( metrics=metrics, metadata=_key_metadata(api_key_metadata, record.api_key), ) elif level == _GROUP_DATE_MODEL: - if record.model: - assign_metric_with_metadata(breakdown.models, record.model, metrics) + assign_metric_with_metadata(breakdown.models, model_bucket(record.model), metrics) elif level == _GROUP_DATE_MODEL_API_KEY: - if record.model and record.api_key and not is_ptu_sentinel: - assign_api_key_breakdown(breakdown.models, record.model, record.api_key, metrics) + if not is_ptu_sentinel: + assign_api_key_breakdown(breakdown.models, model_bucket(record.model), record.api_key, metrics) elif level == _GROUP_DATE_MODEL_GROUP: - if record.model_group: - assign_metric_with_metadata(breakdown.model_groups, record.model_group, metrics) + assign_metric_with_metadata(breakdown.model_groups, model_group_bucket(record.model_group, None), metrics) elif level == _GROUP_DATE_MODEL_GROUP_API_KEY: - if record.model_group and record.api_key and not is_ptu_sentinel: + if not is_ptu_sentinel: assign_api_key_breakdown( breakdown.model_groups, - record.model_group, + model_group_bucket(record.model_group, None), record.api_key, metrics, ) @@ -836,14 +866,14 @@ def _aggregate_grouping_sets_records_sync( provider = record.custom_llm_provider or "unknown" assign_metric_with_metadata(breakdown.providers, provider, provider_metrics) elif level == _GROUP_DATE_PROVIDER_API_KEY: - if record.api_key and not is_ptu_sentinel: + if not is_ptu_sentinel: provider = record.custom_llm_provider or "unknown" assign_api_key_breakdown(breakdown.providers, provider, record.api_key, metrics) elif level == _GROUP_DATE_MCP: if record.mcp_namespaced_tool_name: assign_metric_with_metadata(breakdown.mcp_servers, record.mcp_namespaced_tool_name, metrics) elif level == _GROUP_DATE_MCP_API_KEY: - if record.mcp_namespaced_tool_name and record.api_key: + if record.mcp_namespaced_tool_name and not is_ptu_sentinel: assign_api_key_breakdown( breakdown.mcp_servers, record.mcp_namespaced_tool_name, @@ -854,7 +884,7 @@ def _aggregate_grouping_sets_records_sync( if record.endpoint: assign_metric_with_metadata(breakdown.endpoints, record.endpoint, metrics) elif level == _GROUP_DATE_ENDPOINT_API_KEY: - if record.endpoint and record.api_key: + if record.endpoint and not is_ptu_sentinel: assign_api_key_breakdown(breakdown.endpoints, record.endpoint, record.api_key, metrics) results: Final = [ @@ -1013,8 +1043,8 @@ def _fold_entity_rollups_sync( metrics = _record_to_spend_metrics(row) if row.api_key_rolled: bucket.metrics = metrics - elif row.api_key and row.api_key != PTU_SENTINEL_API_KEY: - bucket.api_key_breakdown[row.api_key] = KeyMetricWithMetadata( + elif row.api_key != PTU_SENTINEL_API_KEY: + bucket.api_key_breakdown[api_key_bucket(row.api_key)] = KeyMetricWithMetadata( metrics=metrics, metadata=_key_metadata(api_key_metadata, row.api_key) ) diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 79e515944ce..1a3445aae52 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -22,6 +22,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( parse_canonical_date_range, raise_public, spend_logs_window, + sum_spend_metrics, ) from litellm.proxy.management_endpoints.daily_activity_scopes import ( AGENT_RESOLVER, @@ -38,6 +39,7 @@ from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, w from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw +from litellm.repositories.daily_activity_buckets import api_key_bucket from litellm.repositories.daily_activity_repository import DailyActivityRepository from litellm.router import Router from litellm.types.proxy.management_endpoints.common_daily_activity import ( @@ -106,29 +108,9 @@ async def _resolved_scope( return result -def _sum_metrics(metrics: Sequence[SpendMetrics]) -> SpendMetrics: - return SpendMetrics( - spend=sum(metric.spend for metric in metrics), - flat_cost=sum(metric.flat_cost for metric in metrics), - prompt_tokens=sum(metric.prompt_tokens for metric in metrics), - completion_tokens=sum(metric.completion_tokens for metric in metrics), - cache_read_input_tokens=sum(metric.cache_read_input_tokens for metric in metrics), - cache_creation_input_tokens=sum(metric.cache_creation_input_tokens for metric in metrics), - compression_saved_tokens=sum(metric.compression_saved_tokens for metric in metrics), - compression_savings_spend=sum(metric.compression_savings_spend for metric in metrics), - prompt_caching_savings_spend=sum(metric.prompt_caching_savings_spend for metric in metrics), - gateway_injected_caching_savings_spend=sum(metric.gateway_injected_caching_savings_spend for metric in metrics), - autorouter_savings_spend=sum(metric.autorouter_savings_spend for metric in metrics), - total_tokens=sum(metric.total_tokens for metric in metrics), - successful_requests=sum(metric.successful_requests for metric in metrics), - failed_requests=sum(metric.failed_requests for metric in metrics), - api_requests=sum(metric.api_requests for metric in metrics), - total_response_time_ms=sum(metric.total_response_time_ms for metric in metrics), - timed_requests=sum(metric.timed_requests for metric in metrics), - ) - - def _key_metadata(api_key: str, metadata: Mapping[str, KeyMetadataRow]) -> KeyMetadata: + if not api_key: + return KeyMetadata(key_alias="Unattributed", key_exists=False) row: Final[KeyMetadataRow | None] = metadata.get(api_key) if row is None: return KeyMetadata() @@ -143,7 +125,7 @@ def _key_metadata(api_key: str, metadata: Mapping[str, KeyMetadataRow]) -> KeyMe def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow]) -> KeySpendActivityRow: return KeySpendActivityRow( - api_key=row.api_key, + api_key=api_key_bucket(row.api_key), metrics=KeySpendMetrics( spend=row.spend, prompt_tokens=row.prompt_tokens, @@ -168,7 +150,7 @@ async def _key_activity_rows( frozenset((resolved_scope.scope.start_date, resolved_scope.scope.end_date)) ) metadata: Final[Mapping[str, KeyMetadataRow]] = await repository.key_metadata( - frozenset(row.api_key for row in rows), + frozenset(row.api_key for row in rows if row.api_key), spend_window, ) return [_key_activity_row(row, metadata) for row in rows] @@ -231,11 +213,11 @@ def _fold_key_metrics(api_key: str, response: SpendAnalyticsPaginatedResponse) - metadata: Final[KeyMetadata] = next( day.breakdown.api_keys[api_key].metadata for day in response.results if api_key in day.breakdown.api_keys ) - return KeyActivityRow(api_key=api_key, metrics=_sum_metrics(metrics), metadata=metadata) + return KeyActivityRow(api_key=api_key, metrics=sum_spend_metrics(metrics), metadata=metadata) def _search_rows(keys: Sequence[str], response: SpendAnalyticsPaginatedResponse) -> list[KeyActivityRow]: - return [row for key in keys if (row := _fold_key_metrics(key, response)) is not None] + return [row for key in keys if (row := _fold_key_metrics(api_key_bucket(key), response)) is not None] def _csv_cell(value: object) -> object: diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index 77246f5dda6..ca1a96f9d84 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -17,6 +17,7 @@ from litellm.proxy.management_endpoints.organization_endpoints import resolve_or from litellm.proxy.management_endpoints.tag_management_endpoints import get_tag_daily_activity_api_key_filter from litellm.proxy.management_endpoints.team_endpoints import resolve_team_daily_activity_scope from litellm.proxy.utils import PrismaClient +from litellm.repositories.daily_activity_buckets import raw_api_key_filter from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable @@ -77,13 +78,7 @@ def _build_scope( ) -> ResolvedScope: start_date: Final[str] = query.start_date or "" end_date: Final[str] = query.end_date or "" - api_keys: Final[tuple[str, ...] | None] = ( - None - if api_key_filter is None or api_key_filter == "" - else (api_key_filter,) - if isinstance(api_key_filter, str) - else tuple(api_key_filter) - ) + api_keys: Final = raw_api_key_filter(api_key_filter) return ResolvedScope( scope=DailyActivityScope( table=resolver.table, diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 6a9a30a7689..ec8ee5ca213 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.utils import PrismaClient, evict_config_param, invalidate_config_param +from litellm.proxy.utils import PrismaClient, invalidate_config_param if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status @@ -48,22 +48,50 @@ FallbackRule = dict[str, list[str]] StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) +_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) def _rule_covers(entry: StoredFallback, model: str) -> bool: return isinstance(entry, dict) and model in entry -async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]: - await evict_config_param(ROUTER_SETTINGS_PARAM) - config: Final = await proxy_config.get_config() - return config.get(ROUTER_SETTINGS_PARAM, {}) +async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None: + await proxy_config.get_config() -def _reject_config_owned_rules(proxy_config: "ProxyConfig", fallback_key: str, rules: Sequence[StoredFallback]) -> None: +async def _stored_router_settings(prisma_client: PrismaClient) -> Mapping[str, JsonValue]: + try: + row: Final = await ConfigRepository(prisma_client, use_writer=True).get_param(ROUTER_SETTINGS_PARAM) + except json.JSONDecodeError: + return {} + if row is None or not isinstance(row.param_value, Mapping): + return {} + return _STORED_ROUTER_SETTINGS.validate_python(row.param_value) + + +def _editable_rules( + proxy_config: "ProxyConfig", stored_settings: Mapping[str, JsonValue], fallback_key: str +) -> list[StoredFallback]: + file_settings: Final = proxy_config.router_settings + if not file_settings.owned_by_config(fallback_key): + return _STORED_FALLBACKS.validate_python(stored_settings.get(fallback_key) or []) + file_rules: Final = file_settings.get(fallback_key) + return _STORED_FALLBACKS.validate_python(file_rules if isinstance(file_rules, list) else []) + + +async def _write_rules( + prisma_client: PrismaClient, + proxy_config: "ProxyConfig", + stored_settings: Mapping[str, JsonValue], + fallback_key: str, + rules: Sequence[StoredFallback], +) -> None: proxy_config.reject_config_owned_writes( section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)} ) + if proxy_config.router_settings.owned_by_config(fallback_key): + return + await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules}) async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None: @@ -162,7 +190,8 @@ async def create_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + await _reload_config_file_ownership(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -172,7 +201,7 @@ async def create_fallback( fallback_key = "content_policy_fallbacks" # Get existing fallbacks - existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or []) + existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) # Update or add the fallback configuration fallback_updated = False @@ -187,12 +216,7 @@ async def create_fallback( # Add new fallback existing_fallbacks.append({data.model: data.fallback_models}) - _reject_config_owned_rules(proxy_config, fallback_key, existing_fallbacks) - - # Update router settings - router_settings[fallback_key] = existing_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks) # Update the in-memory router configuration setattr(llm_router, fallback_key, existing_fallbacks) @@ -320,7 +344,8 @@ async def delete_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + await _reload_config_file_ownership(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -330,7 +355,7 @@ async def delete_fallback( fallback_key = "content_policy_fallbacks" # Get existing fallbacks - existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or []) + existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) updated_fallbacks: Final = [rule for rule in existing_fallbacks if not _rule_covers(rule, model)] @@ -340,12 +365,7 @@ async def delete_fallback( detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"}, ) - _reject_config_owned_rules(proxy_config, fallback_key, updated_fallbacks) - - # Update router settings - router_settings[fallback_key] = updated_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks) # Update the in-memory router configuration setattr(llm_router, fallback_key, updated_fallbacks) diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index c12a90ed6a2..ea7810a618c 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -31,6 +31,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( get_daily_activity, ) from litellm.proxy.management_helpers.utils import handle_budget_for_entity +from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ( DailyTagSpendRepository, @@ -198,16 +199,17 @@ async def get_tag_daily_activity_api_key_filter( user_api_key_dict: UserAPIKeyAuth, requested_api_key: str | None, ) -> str | list[str] | None: + selected_key: Final = api_key_from_bucket(requested_api_key) if requested_api_key else None user_role: Final = user_api_key_dict.user_role if user_api_key_has_admin_view(user_api_key_dict) or (user_role is None or not user_role.is_internal_user_role): - return requested_api_key + return [selected_key] if selected_key is not None else None scoped_api_keys: Final = await _get_internal_user_api_keys( prisma_client=prisma_client, user_api_key_dict=user_api_key_dict, ) - if requested_api_key is not None: - return requested_api_key if requested_api_key in scoped_api_keys else [] + if selected_key is not None: + return [selected_key] if selected_key in scoped_api_keys else [] return scoped_api_keys diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 50b700e712d..b712ba42a6a 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -226,6 +226,7 @@ from litellm.proxy.management_helpers.utils import ( from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.chunked_in import delete_many_in, find_many_in +from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ( @@ -7329,11 +7330,12 @@ async def resolve_team_daily_activity_scope( if not user_api_keys: user_api_keys = [] + selected_key: Final = api_key_from_bucket(api_key) if api_key else None final_api_key_filter: str | list[str] | None if user_api_keys is None: - final_api_key_filter = api_key - elif api_key: - final_api_key_filter = api_key if api_key in user_api_keys else [] + final_api_key_filter = [selected_key] if selected_key is not None else None + elif selected_key is not None: + final_api_key_filter = [selected_key] if selected_key in user_api_keys else [] else: final_api_key_filter = user_api_keys diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 22c4f27d94a..a980b9c33ff 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -16,6 +16,7 @@ import json import os import re import secrets +from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from html import escape @@ -42,7 +43,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator +from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator import litellm from litellm._internal_context import with_service_target @@ -106,6 +107,7 @@ from litellm.proxy.common_utils.html_forms.default_credentials_hint import shoul from litellm.proxy.common_utils.html_forms.jwt_display_template import ( jwt_display_template, ) +from litellm.proxy.common_utils.html_forms.native_client_consent import team_select_field from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -184,6 +186,8 @@ _CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS: Final = 60 _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: Final = 30 _CLI_SSO_USER_CODE_ALPHABET: Final = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" _CLI_SSO_LOGIN_ID_RE: Final = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$") +CLI_SSO_BROWSER_TEAM_CHOICE: Final = "browser_team_choice" +CLI_SSO_CHOSEN_TEAM_ID: Final = "chosen_team_id" _CLI_SSO_USER_CODE_RE = re.compile(rf"^[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}-[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}$") _CLI_SSO_SCALAR_TYPES: Final = (str, int, float, bool) _CLI_SSO_DEST_KEY_RE: Final = re.compile(r"^[A-Za-z0-9_.-]+$") @@ -670,6 +674,7 @@ def _render_cli_sso_verification_page( verify_url: str, browser_complete_token: str, prefill_user_code: str | None = None, + team_choices: Sequence[tuple[str, str]] = (), ) -> str: escaped_verify_url: Final = escape(verify_url, quote=True) escaped_browser_complete_token: Final = escape(browser_complete_token, quote=True) @@ -716,6 +721,15 @@ def _render_cli_sso_verification_page( letter-spacing: 0.08em; text-transform: uppercase; }} + select {{ + box-sizing: border-box; + width: 100%; + padding: 12px; + margin-bottom: 18px; + border: 1px solid #cbd5e1; + border-radius: 6px; + font-size: 16px; + }} button {{ margin-top: 16px; width: 100%; @@ -735,6 +749,7 @@ def _render_cli_sso_verification_page(

{instructions}

+ {team_select_field(team_choices, require_choice=True)} @@ -823,13 +838,46 @@ async def cli_sso_complete(request: Request, login_id: str): ): raise HTTPException(status_code=400, detail="Invalid verification code") - flow["user_code_verified"] = True - _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) + team_choice_flow: Final = _CliSsoTeamChoiceFlow.model_validate(flow) + chosen_team_id: Final = _browser_chosen_team_id(team_choice_flow, form_values) + if chosen_team_id is not None and chosen_team_id not in team_choice_flow.teams: + raise HTTPException(status_code=400, detail="Choose one of the teams you belong to") + + _set_cli_sso_flow( + login_id=login_id, + cache=cli_sso_session_cache, + flow={ + **flow, + "user_code_verified": True, + CLI_SSO_CHOSEN_TEAM_ID: chosen_team_id, + "session_data": ( + flow["session_data"] + if chosen_team_id is None + else { + **flow["session_data"], + "teams": [chosen_team_id, *(team for team in team_choice_flow.teams if team != chosen_team_id)], + } + ), + }, + ) html_content: Final = render_cli_sso_success_page() return HTMLResponse(content=html_content, status_code=200) +class _CliSsoTeamChoiceFlow(BaseModel): + model_config = ConfigDict(frozen=True) + + browser_team_choice: bool = Field(default=False, validation_alias=CLI_SSO_BROWSER_TEAM_CHOICE) + teams: tuple[str, ...] = Field(default=(), validation_alias=AliasPath("session_data", "teams")) + + +def _browser_chosen_team_id(flow: _CliSsoTeamChoiceFlow, form_values: Mapping[str, Sequence[str]]) -> str | None: + if not flow.browser_team_choice: + return None + return (form_values.get("team_id") or [""])[0] or None + + def normalize_email(email: str | None) -> str | None: """ Normalize email address to lowercase for consistent storage and comparison. @@ -2416,6 +2464,18 @@ def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str return [detail.team_id for detail in team_details if detail.team_id is not None] +def _browser_team_choices( + flow: _CliSsoTeamChoiceFlow, team_details: Sequence[CliSsoTeamDetail] +) -> tuple[tuple[str, str], ...]: + if not flow.browser_team_choice: + return () + labeled: Final = tuple( + (detail.team_id, detail.team_alias or detail.team_id) for detail in team_details if detail.team_id is not None + ) + label_counts: Final = Counter(label for _, label in labeled) + return tuple((team_id, label if label_counts[label] == 1 else f"{label} ({team_id})") for team_id, label in labeled) + + def selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> CliSsoTeamDetail | None: """``None`` means the team's grants are unknown. An empty grant is a real value meaning unrestricted, so an unknown one must not be minted as empty.""" @@ -2484,6 +2544,7 @@ async def _complete_cli_sso_callback_session( request.scope.get("litellm_microsoft_interactive_subject"), resolved_user_id, prisma_client ) resolved_teams: Final = _cli_sso_session_teams(team_details) + team_choices: Final = _browser_team_choices(_CliSsoTeamChoiceFlow.model_validate(flow), team_details) attribution_metadata: Final = build_cli_sso_attribution_metadata(result=result) if attribution_metadata: await _persist_cli_sso_user_metadata( @@ -2521,6 +2582,7 @@ async def _complete_cli_sso_callback_session( verify_url=verify_url, browser_complete_token=browser_complete_token, prefill_user_code=prefill_user_code, + team_choices=team_choices, ), status_code=200, ) diff --git a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py index c417187396f..6c6be219a92 100644 --- a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py +++ b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py @@ -43,6 +43,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span from litellm.proxy.mcp_server.oauth_utils import canonicalize_url_identity from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import MCPServerRepository # The actor the removed discovery write-back stamped rows with. _DISCOVERY_ACTOR: Final = "mcp_oauth_discovery" @@ -106,7 +107,7 @@ def _is_stamped_issuer_row(row: _MCPServerRow) -> bool: async def backfill_discovery_stamped_issuers(prisma_client: PrismaClient) -> int: """Clear gateway-written issuer stamps, returning the number of rows healed.""" - candidate_rows: Final[list[_MCPServerRow]] = await prisma_client.db.litellm_mcpservertable.find_many( + candidate_rows: Final = await MCPServerRepository(prisma_client).table.find_many( where={ "updated_by": _DISCOVERY_ACTOR, "auth_type": {"in": list(_AUTH_TYPES_WITH_ISSUER_ANCHORING)}, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 972ec18d487..f96fab75bd1 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -68,6 +68,7 @@ from litellm.llms.deepgram.common_utils import ( from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.openai.common_utils import OpenAIError as LiteLLMOpenAIError +from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_api_resource from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -3530,6 +3531,10 @@ async def deepgram_listen_websocket_route( ) +_OPENAI_BETA_HEADER: Final = "openai-beta" +_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"}) + + class BaseOpenAIPassThroughHandler: @staticmethod async def base_openai_pass_through_handler( @@ -3540,7 +3545,7 @@ class BaseOpenAIPassThroughHandler: base_target_url: str, api_key: str | None, custom_llm_provider: litellm.LlmProviders, - extra_headers: dict | None = None, + extra_headers: Mapping[str, str] | None = None, ): encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -3565,7 +3570,11 @@ class BaseOpenAIPassThroughHandler: endpoint=endpoint, target=str(updated_url), custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( - api_key=api_key, request=request, extra_headers=extra_headers + api_key=api_key, + endpoint=endpoint, + request=request, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, ), is_streaming_request=is_streaming_request, custom_llm_provider=( @@ -3587,27 +3596,41 @@ class BaseOpenAIPassThroughHandler: _base_openai_pass_through_handler = base_openai_pass_through_handler @staticmethod - def _append_openai_beta_header(headers: dict, request: Request) -> dict: - """ - Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request - """ - if RouteChecks.is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: - headers["OpenAI-Beta"] = "assistants=v2" - return headers + def _openai_beta(endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders) -> str | None: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and openai_api_resource(endpoint) in _CLIENT_OPENAI_BETA_RESOURCES + ): + return request.headers.get(_OPENAI_BETA_HEADER) or ( + "agents=v1" if is_openai_agents_api_path(endpoint) else None + ) + return "assistants=v2" if RouteChecks.is_assistants_api_request(request) else None @staticmethod - def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict: - base_headers = {} - if api_key is not None: - base_headers = { - "authorization": f"Bearer {api_key}", - "api-key": f"{api_key}", - } - if extra_headers is not None: - base_headers.update(extra_headers) + def _append_openai_beta_header( + headers: Mapping[str, str], endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders + ) -> dict[str, str]: + if any(name.lower() == _OPENAI_BETA_HEADER for name in headers): + return {**headers} + openai_beta: Final = BaseOpenAIPassThroughHandler._openai_beta(endpoint, request, custom_llm_provider) + return {**headers} if openai_beta is None else {**headers, _OPENAI_BETA_HEADER: openai_beta} + + @staticmethod + def _assemble_headers( + api_key: str | None, + endpoint: str, + request: Request, + custom_llm_provider: litellm.LlmProviders, + extra_headers: Mapping[str, str] | None = None, + ) -> dict[str, str]: + api_key_headers: Final[Mapping[str, str]] = ( + {} if api_key is None else {"authorization": f"Bearer {api_key}", "api-key": api_key} + ) return BaseOpenAIPassThroughHandler._append_openai_beta_header( - headers=base_headers, + headers={**api_key_headers, **(extra_headers or {})}, + endpoint=endpoint, request=request, + custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a11b0be3983..4eeb9b26174 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,10 +1,12 @@ from datetime import datetime from typing import Final +from urllib.parse import urlparse import httpx import litellm from litellm import stream_chunk_builder +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -16,7 +18,7 @@ from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig, CohereEmbeddingRowShape from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, @@ -29,6 +31,12 @@ from litellm.types.utils import ( from .base_passthrough_logging_handler import BasePassthroughLoggingHandler +COHERE_EMBED_PATHS: Final = ("/v1/embed", "/v2/embed") + + +def is_cohere_embed_route(url_route: str) -> bool: + return urlparse(url_route).path.rstrip("/").endswith(COHERE_EMBED_PATHS) + class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): @property @@ -82,28 +90,24 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle Cohere passthrough logging with route detection and cost tracking. """ - # Check if this is an embed endpoint - if "/v1/embed" in url_route and "/v1/embeddings" not in url_route: + if is_cohere_embed_route(url_route): model: Final = request_body.get("model", response_body.get("model", "")) try: - cohere_embed_config: Final = CohereEmbeddingConfig() - litellm_model_response = litellm.EmbeddingResponse() handler_instance: Final = CoherePassthroughLoggingHandler() input_texts = request_body.get("texts", []) if not input_texts: input_texts = request_body.get("input", []) - # Transform the response - litellm_model_response = cohere_embed_config.transform_response( + litellm_model_response: Final = CohereEmbeddingConfig().transform_native_embed_response( response=httpx_response, api_key="", logging_obj=logging_obj, data=request_body, - model_response=litellm_model_response, model=model, encoding=litellm.encoding, input=input_texts, + row_shape=CohereEmbeddingRowShape.TYPED, ) # Calculate cost using LiteLLM's cost calculator @@ -153,20 +157,11 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): "result": litellm_model_response, "kwargs": kwargs, } - except Exception: - # For other routes (e.g., /v2/chat), fall back to chat handler - return super().passthrough_chat_handler( - httpx_response=httpx_response, - response_body=response_body, - logging_obj=logging_obj, - url_route=url_route, - result=result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - request_body=request_body, - **kwargs, + except Exception as e: + verbose_proxy_logger.warning( + "Could not price the Cohere passthrough embed response, logging it without a cost: %s", e ) + return {"result": None, "kwargs": kwargs} # For non-embed routes (e.g., /v2/chat), fall back to chat handler return super().passthrough_chat_handler( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c410150df70..b98dd42e3de 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,7 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.llms.openai.common_utils import is_openai_agents_api_path from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -38,7 +40,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, PassthroughCallTypes +from litellm.types.utils import ( + EmbeddingResponse, + ImageResponse, + LlmProviders, + PassthroughCallTypes, + StandardPassThroughResponseObject, +) from litellm.utils import ModelResponse, TextCompletionResponse, convert_to_model_response_object # Hostnames that route to OpenAI-compatible APIs. @@ -121,6 +129,52 @@ def _responses_sub_route(path: str) -> tuple[str, ...] | None: return None if anchor is None else tuple(segments[anchor + 1 :]) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +class _AgentConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + model: str + + +class _AgentsApiBody(BaseModel): + model_config = ConfigDict(frozen=True) + + agent: _AgentConfig + + +class _AgentsApiSessionEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + session: _AgentsApiBody + + +_AGENTS_API_BODY: Final = TypeAdapter(_AgentsApiBody | _AgentConfig) + + +def _agent_model(body: Mapping[str, object]) -> str | None: + try: + parsed: Final = _AGENTS_API_BODY.validate_python(body) + except ValidationError: + return None + return parsed.agent.model if isinstance(parsed, _AgentsApiBody) else parsed.model + + +def _session_event_model(chunk: str) -> str | None: + event: Final = BaseModelResponseIterator.string_to_dict_parser(str_line=chunk) + if event is None: + return None + try: + return _AgentsApiSessionEvent.model_validate(event).session.agent.model + except ValidationError: + return None + + +def _streamed_agent_model(all_chunks: Sequence[str]) -> str | None: + return next((model for chunk in all_chunks if (model := _session_event_model(chunk)) is not None), None) + + def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(part, Mapping) or part.get("type") != "image_url": return False @@ -220,14 +274,44 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return "billed" @staticmethod - def _label_unbilled_responses_call( + def is_openai_agents_route(url_route: str) -> bool: + return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) + + @staticmethod + def _unbilled_call_model( + url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object] + ) -> str | None: + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return _agent_model(response_body) or _agent_model(request_body) + response_model: Final = response_body.get("model") + if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(response_model, str): + return response_model + return None + + @staticmethod + def _label_model(logging_obj: LiteLLMLoggingObj, model: str | None) -> None: + if model is not None: + logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + + @staticmethod + def _label_unbilled_call( logging_obj: LiteLLMLoggingObj, url_route: str, + request_body: Mapping[str, object], response_body: Mapping[str, object], ) -> None: - model: Final = response_body.get("model") - if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str): - logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + OpenAIPassthroughLoggingHandler._label_model( + logging_obj, OpenAIPassthroughLoggingHandler._unbilled_call_model(url_route, request_body, response_body) + ) + + @staticmethod + def _log_agents_stream( + litellm_logging_obj: LiteLLMLoggingObj, request_body: Mapping[str, object], all_chunks: Sequence[str] + ) -> PassThroughEndpointLoggingTypedDict: + OpenAIPassthroughLoggingHandler._label_model( + litellm_logging_obj, _streamed_agent_model(all_chunks) or _agent_model(request_body) + ) + return {"result": StandardPassThroughResponseObject(response="\n".join(all_chunks)), "kwargs": {}} @staticmethod def is_openai_embeddings_route(url_route: str) -> bool: @@ -397,7 +481,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_responses: Final = responses_billing != "unbilled" if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): - OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body) + OpenAIPassthroughLoggingHandler._label_unbilled_call( + logging_obj, + url_route, + request_body=_STR_MAPPING.validate_python(request_body), + response_body=_STR_MAPPING.validate_python(response_body), + ) return { "result": None, "kwargs": kwargs, @@ -676,6 +765,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle logging for collected OpenAI streaming chunks with cost tracking. """ + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return OpenAIPassthroughLoggingHandler._log_agents_stream( + litellm_logging_obj, _STR_MAPPING.validate_python(request_body), all_chunks + ) try: is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) terminal_response: Final = ( diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 1d0b3d00434..739d4b3a01a 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -12,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.secret_managers.main import ( + GOOGLE_OIDC_SECRET_PREFIX, UnresolvableSecretReference, get_secret_str, resolve_local_oidc_reference, @@ -47,6 +48,14 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str | def _resolve_oidc_reference(credential: str | None) -> str | None: if credential is None: return None + if credential.startswith(GOOGLE_OIDC_SECRET_PREFIX): + raise HTTPException( + status_code=401, + detail=( + "pass-through credential oidc/google secret reference could not be resolved: pass-through routes " + "do not fetch Google ID tokens. Use a static key, oidc/file/ or oidc/env/." + ), + ) match resolve_local_oidc_reference(credential): case UnresolvableSecretReference(kind=kind, reason=reason): raise HTTPException( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..27e31c8f74d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -22,6 +22,7 @@ from .llm_provider_handlers.assembly_passthrough_logging_handler import ( ) from .llm_provider_handlers.cohere_passthrough_logging_handler import ( CoherePassthroughLoggingHandler, + is_cohere_embed_route, ) from .llm_provider_handlers.cursor_passthrough_logging_handler import ( CursorPassthroughLoggingHandler, @@ -91,7 +92,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages", "/v1/messages/batches"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat"] self.assemblyai_passthrough_logging_handler = AssemblyAIPassthroughLoggingHandler() # Langfuse @@ -536,13 +537,7 @@ class PassThroughEndpointLogging: return False def is_cohere_route(self, url_route: str) -> bool: - for route in self.TRACKED_COHERE_ROUTES: - if route not in url_route: - continue - if route == "/v1/embed" and "/v1/embeddings" in url_route: - continue - return True - return False + return is_cohere_embed_route(url_route) or any(route in url_route for route in self.TRACKED_COHERE_ROUTES) def is_assemblyai_route(self, url_route: str): parsed_url: Final = urlparse(url_route) @@ -645,6 +640,7 @@ class PassThroughEndpointLogging: or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) ) def _set_cost_per_request( diff --git a/litellm/proxy/plugin_routes.py b/litellm/proxy/plugin_routes.py index 1dfe5cbc6bb..cfbcfefe9e8 100644 --- a/litellm/proxy/plugin_routes.py +++ b/litellm/proxy/plugin_routes.py @@ -31,6 +31,8 @@ from typing import Final from cryptography.fernet import Fernet, InvalidToken from fastapi import APIRouter, Depends, HTTPException, Request, Response +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import PluginConfig, SpecialHeaders, UserAPIKeyAuth @@ -39,6 +41,18 @@ from litellm.types.llms.custom_http import httpxSpecialProvider router: Final = APIRouter() + +@with_config(ConfigDict(extra="allow", strict=True)) +class PluginSessionClaim(TypedDict): + plugin: NotRequired[ReadOnly[object]] + user_id: NotRequired[ReadOnly[str]] + user_role: NotRequired[ReadOnly[str]] + exp: NotRequired[ReadOnly[int | float | str]] + + +_PLUGIN_SESSION_CLAIM: Final = TypeAdapter(PluginSessionClaim) + + # Hop-by-hop headers (RFC 7230) and the litellm session cookie — never forwarded # to a plugin backend. Credential headers are added on top per-request from the # canonical SpecialHeaders set so the plugin only ever authenticates via its own @@ -155,7 +169,7 @@ def issue_plugin_session_claim(plugin_name: str, user_id: str | None, user_role: return _plugin_fernet(plugin_name).encrypt(json.dumps(claim).encode()).decode() -def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: +def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> PluginSessionClaim: """Verify and decode a plugin session claim. Raises ValueError if the HMAC is invalid, the audience is wrong, or @@ -163,7 +177,7 @@ def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: """ try: raw: Final = _plugin_fernet(plugin_name).decrypt(ciphertext.encode(), ttl=_CLAIM_TTL_SECONDS) - claim: Final = json.loads(raw) + claim: Final = _PLUGIN_SESSION_CLAIM.validate_python(json.loads(raw)) except (InvalidToken, Exception) as exc: raise ValueError("Invalid, tampered, or expired plugin session claim") from exc diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 59d15b0ba0a..2d0e9d25368 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -11,7 +11,7 @@ from collections.abc import Mapping, Sequence from itertools import chain from typing import TYPE_CHECKING, Any, Final, Optional -from litellm._logging import print_startup_message, verbose_proxy_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_proxy_logger from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_validator import PolicyValidator @@ -93,7 +93,7 @@ def _print_policies_on_startup( ) else: print_startup_message("\n".join((header, *policy_lines))) - print_startup_message( + print_startup_warning( f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}\n", ) sys.stdout.flush() diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 909f6de40cd..f71d1960e24 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -10,16 +10,17 @@ import sys import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Annotated, Any, Final import click import httpx from click.core import ParameterSource from dotenv import load_dotenv -from pydantic import ConfigDict +from pydantic import BeforeValidator, ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm -from litellm._logging import print_startup_message +from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, @@ -205,7 +206,7 @@ def deprecated_v2_flag_passed_on_cli() -> bool: def _print_enforce_prisma_migration_check_notice() -> None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " "deprecated and has no effect, because the proxy always exits " "when database setup fails at startup. You can safely remove it.\033[0m", @@ -227,6 +228,36 @@ def _print_or_hold_enforce_prisma_migration_check_notice( return print_once +def _empty_as_none(value: object) -> object: + return value or None + + +class _ConfigStartupOutputSettings(TypedDict, total=False): + litellm_settings: ReadOnly[Annotated[StartupOutputSettings | None, BeforeValidator(_empty_as_none)]] + + +_CONFIG_STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(_ConfigStartupOutputSettings) + + +def _apply_config_suppress_startup_output(litellm_settings: StartupOutputSettings | None) -> None: + if litellm_settings is None or litellm_settings.get("suppress_startup_output") is not True: + return + litellm.suppress_startup_output = True + os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true" + + +def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | None) -> None: + """Turn on the config's json_logs and suppress_startup_output before any worker starts. Workers that + uvicorn or granian spawn import litellm afresh and read both from the env at import, so they log JSON + and stay quiet from their first line instead of from when load_config reaches the key.""" + _apply_config_suppress_startup_output(litellm_settings) + if litellm_settings is None or litellm_settings.get("json_logs") is not True: + return + litellm.json_logs = True + os.environ["JSON_LOGS"] = "true" + litellm.turn_on_json() + + class ProxyInitializationHelpers: @staticmethod def echo_litellm_version(): @@ -245,18 +276,19 @@ class ProxyInitializationHelpers: @staticmethod def run_migration_data_check(use_migrate: bool) -> int: - """`--check_migrations_against_existing_data`: the pre-migration row check against DATABASE_URL, nothing applied. - 0 when the pending schema changes fit the rows, 2 with the refusal when they do not.""" + """`--check_migrations_against_existing_data`: the pending migrations applied to DATABASE_URL inside one + transaction that is rolled back, so Postgres reports what the upgrade would hit. 0 when they all apply, + 2 with Postgres's error when one does not.""" if os.getenv("DATABASE_URL") is None: raise click.UsageError("--check_migrations_against_existing_data requires DATABASE_URL") from litellm.proxy.db.prisma_client import PrismaManager - refusal: Final[str | None] = PrismaManager.migration_data_check_report(use_migrate=use_migrate) # pyright: ignore[reportUnknownVariableType] # litellm-proxy-extras is an optional install - if refusal is None: - print("LiteLLM Proxy: the pending migrations and indexes can be applied to the rows this database has.") + ok, summary = PrismaManager.migration_dry_run_summary(use_migrate=use_migrate) + if ok: + print(f"LiteLLM Proxy: {summary}") return 0 print( - f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {refusal}\033[0m", file=sys.stderr, flush=True + f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {summary}\033[0m", file=sys.stderr, flush=True ) return 2 @@ -373,7 +405,7 @@ class ProxyInitializationHelpers: if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -396,7 +428,7 @@ class ProxyInitializationHelpers: import uvicorn if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -404,7 +436,7 @@ class ProxyInitializationHelpers: if "limit_max_requests_jitter" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["limit_max_requests_jitter"] = jitter else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " f"requires uvicorn>=0.41.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -571,12 +603,12 @@ class ProxyInitializationHelpers: print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n") if max_requests_before_restart is not None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM: --max_requests_before_restart is not supported by Granian " "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n", ) if ciphers is not None: - print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") + print_startup_warning("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") kwargs: Final[dict[str, object]] = { "target": "litellm.proxy.proxy_server:app", @@ -698,7 +730,7 @@ class ProxyInitializationHelpers: gunicorn_options["max_requests"] = max_requests_before_restart if max_requests_before_restart_jitter is not None: if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -739,7 +771,7 @@ class ProxyInitializationHelpers: with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - print_startup_message(f""" + print_startup_warning(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` """) @@ -1028,8 +1060,9 @@ class ProxyInitializationHelpers: "--check_migrations_against_existing_data", is_flag=True, default=False, - help="Check, read-only, that the rows in DATABASE_URL can take this version's pending migrations and indexes, " - "print what would block them, then exit without applying anything. Exit code 2 when something blocks.", + help="Dry-run this version's pending migrations on DATABASE_URL: the real SQL inside one transaction that is " + "rolled back, so Postgres reports the row or schema that would make the upgrade fail, then exit. Holds the same " + "locks the upgrade will. Exit code 2 when a migration fails.", ) @click.option( "--keepalive_timeout", @@ -1325,6 +1358,19 @@ def run_server( os.environ[k] = v litellm_settings = None + env_config_file: Final = os.environ.get("CONFIG_FILE_PATH") + if ( + env_config_file + and os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is None + and ProxyConfig().is_yaml(config_file_path=env_config_file) + ): + import asyncio + + _apply_config_suppress_startup_output( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python( + asyncio.run(ProxyConfig().read_config_file(config_file_path=env_config_file)) + ).get("litellm_settings") + ) if config is not None: """ Allow user to pass in db url via config @@ -1342,17 +1388,9 @@ def run_server( ### LITELLM SETTINGS ### litellm_settings = _config.get("litellm_settings", None) - if ( - litellm_settings is not None - and "json_logs" in litellm_settings - and litellm_settings["json_logs"] is True - ): - import litellm - - litellm.json_logs = True - os.environ["JSON_LOGS"] = "true" - - litellm.turn_on_json() + _apply_config_logging_settings( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python(_config).get("litellm_settings") + ) if held_prisma_notice is not None: held_prisma_notice() ### GENERAL SETTINGS ### @@ -1450,7 +1488,9 @@ def run_server( _bad_scheme = unsupported_db_scheme(_candidate_url) if _bad_scheme is not None: print( - f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1553,7 +1593,7 @@ def run_server( env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"), ) if deprecated_v2_flag_passed_on_cli() and use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " "deprecated and has no effect, because the v2 migration " "resolver is now the default. You can safely remove it. To " @@ -1561,7 +1601,7 @@ def run_server( "--use_legacy_migration_resolver.\033[0m", ) if not use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration " "resolver. It performs the diff-and-force recovery that can " "cause schema thrashing during rolling deploys where two " @@ -1583,22 +1623,26 @@ def run_server( # issues, and any `prisma db push` against a # partitioned LiteLLM_SpendLogs. print( - f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", + terminal_text(f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m"), file=sys.stderr, flush=True, ) sys.exit(2) if not setup_ok: print( - "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " - "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " + "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + ) ) sys.exit(1) else: print( - "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " - "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " - "or run a shipped LiteLLM image.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " + "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " + "or run a shipped LiteLLM image.\033[0m" + ) ) sys.exit(1) pgbouncer_settings: Final = PgBouncerSettings() @@ -1609,8 +1653,10 @@ def run_server( ) if isinstance(pooled_database_url, PgBouncerError): print( - f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " - f"could not start: {pooled_database_url.reason}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1675,7 +1721,7 @@ def run_server( run_gunicorn=run_gunicorn is True, ) if single_worker_warning is not None: - print_startup_message(f"\033[1;33m{single_worker_warning}\033[0m\n") + print_startup_warning(f"\033[1;33m{single_worker_warning}\033[0m\n") if run_gunicorn is False and run_hypercorn is False and run_granian is False: if limit_concurrency is not None: uvicorn_args["limit_concurrency"] = limit_concurrency diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5e37ff1ba65..65de7a3c472 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -79,6 +79,7 @@ from litellm.constants import ( DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL, DEFAULT_SHARED_HEALTH_CHECK_TTL, DEFAULT_SLACK_ALERTING_THRESHOLD, + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, INTERNAL_CALLBACK_NAMES, LANGFUSE_SHUTDOWN_FLUSH_TIMEOUT_MILLIS, LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS, @@ -98,6 +99,7 @@ from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, initialize_sentry, ) +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.token_counter import ( @@ -207,7 +209,7 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential +from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential if TYPE_CHECKING: from aiohttp import ClientSession @@ -310,6 +312,7 @@ import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target from litellm._logging import ( + STARTUP_OUTPUT_SETTINGS, print_startup_message, redact_string, verbose_proxy_logger, @@ -1273,15 +1276,44 @@ async def _flush_spend_logs_queue_on_shutdown() -> None: async def _flush_spend_on_shutdown() -> None: + from litellm.proxy.utils import update_daily_tag_spend + await flush_spend_counters_on_shutdown() + if prisma_client is not None: + await update_daily_tag_spend(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) await _flush_spend_logs_queue_on_shutdown() async def _flush_spend_around_logging_drain_on_shutdown( flush_spend: Callable[[], Awaitable[None]] = _flush_spend_on_shutdown, + *, + worker: LoggingWorker = GLOBAL_LOGGING_WORKER, + clock: Callable[[], float] = time.monotonic, + drain_tasks: Callable[[float], Awaitable[None]] = drain_logging_tasks, ) -> None: await flush_spend() - await drain_logging_tasks() + remaining: Final = GracefulShutdownManager.remaining(clock=clock) + callback_timeout: Final = max(0.0, remaining - GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS) + verbose_proxy_logger.info( + "Draining success callbacks before the final spend flush remaining_s=%.3f callback_timeout_s=%.3f", + remaining, + callback_timeout, + ) + + async def drain_callbacks() -> None: + await drain_tasks(callback_timeout) + await worker.flush() + + try: + await asyncio.wait_for(drain_callbacks(), timeout=callback_timeout) + except asyncio.TimeoutError: + verbose_proxy_logger.warning( + "Success callback shutdown budget exhausted with %d unfinished callbacks; accounting may be incomplete", + worker.pending_count, + ) + finally: + await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock)) + await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock)) await flush_spend() @@ -1444,6 +1476,13 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries: @asynccontextmanager async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]: + with GracefulShutdownManager.capture_sigterm(): + async with _proxy_startup_event(app) as state: + yield state + + +@asynccontextmanager +async def _proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]: global \ prisma_client, \ master_key, \ @@ -5805,6 +5844,12 @@ class ProxyConfig: # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") return config + async def read_config_file(self, config_file_path: str) -> Mapping[str, object]: + """The config file with its includes merged, without the bucket, database and secret steps of get_config.""" + return await self._process_includes( + config=self._load_yaml_file(config_file_path), config_file_path=os.path.abspath(config_file_path) + ) + async def _process_includes(self, config: dict, config_file_path: str) -> dict: """ Process includes by appending their contents to the main config @@ -6429,13 +6474,8 @@ class ProxyConfig: search_tools_parsed: Final[list[SearchToolTypedDict]] = [] - print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") - for search_tool in search_tools_raw: - # Display loaded search tool search_tool_name = search_tool.get("search_tool_name", "") - search_provider = search_tool.get("litellm_params", {}).get("search_provider", "") - print_startup_message(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # Handle os.environ/ variables in litellm_params litellm_params = search_tool.get("litellm_params", {}) @@ -6605,13 +6645,15 @@ class ProxyConfig: from litellm.types.integrations.prometheus import validate_caller_identity_settings validate_caller_identity_settings(litellm_settings) + if STARTUP_OUTPUT_SETTINGS.validate_python(litellm_settings).get("suppress_startup_output") is True: + litellm.suppress_startup_output = True # ANSI escape code for blue text blue_color_code: Final = "\033[94m" reset_color_code: Final = "\033[0m" for key, value in litellm_settings.items(): if key == "cache" and value is True: - print_startup_message(f"{blue_color_code}\nSetting Cache on Proxy{reset_color_code}") + print_startup_message(f"\n{blue_color_code}Setting Cache on Proxy{reset_color_code}") from litellm.caching.caching import Cache cache_params = {} @@ -6926,6 +6968,8 @@ class ProxyConfig: setattr(litellm, key, value) elif key == "drop_params": litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger) + elif key == "suppress_startup_output": + continue else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", @@ -6972,6 +7016,7 @@ class ProxyConfig: for name in ( "vector_store_deny_by_default", "search_tool_deny_by_default", + "claude_code_marketplace_auth_required", "vertex_ai_pass_through_drop_anthropic_betas", "request_tag_mappings", ) @@ -7281,6 +7326,13 @@ class ProxyConfig: ## SEARCH TOOLS SETTINGS search_tools: Final[list[SearchToolTypedDict] | None] = self.parse_search_tools(config) + if search_tools: + print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") + for tool in search_tools: + print_startup_message( + f"\033[32m {tool.get('search_tool_name', '')} " + f"({tool.get('litellm_params', {}).get('search_provider', '')})\033[0m" + ) ## SANDBOX TOOLS SETTINGS from litellm.sandbox.sandbox_tools import register_sandbox_tools @@ -9683,12 +9735,9 @@ async def initialize( master_key, \ user_custom_auth, \ prisma_client - from litellm.proxy.common_utils.banner import LITELLM_BANNER, show_banner + from litellm.proxy.common_utils.banner import LITELLM_BANNER - if litellm.json_logs: - print_startup_message(LITELLM_BANNER) - else: - show_banner() + print_startup_message(f"\n{LITELLM_BANNER}\n") if os.getenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "").lower() != "true": generate_feedback_box() user_model = model @@ -14753,10 +14802,10 @@ def _provider_count_outcome( return result -def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: +async def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params) try: - load_credentials_from_list(call_params) + return {**call_params, **(await acredential_updates(call_params))} except litellm.AuthenticationError as e: raise ProxyException( message=e.message, @@ -14764,13 +14813,19 @@ def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: param="model", code=401, ) from e - return call_params + except litellm.ServiceUnavailableError as e: + raise ProxyException( + message=e.message, + type="service_unavailable", + param="model", + code=503, + ) from e -def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None: +async def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None: if deployment is None: return None - return {**deployment, "litellm_params": _with_resolved_api_key(deployment.get("litellm_params", {}))} + return {**deployment, "litellm_params": await _with_resolved_api_key(deployment.get("litellm_params", {}))} _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -14833,7 +14888,7 @@ async def _try_provider_token_count( model_to_use=model_to_use or "", messages=messages, contents=contents, - deployment=_with_resolved_credentials(deployment_in_region), + deployment=await _with_resolved_credentials(deployment_in_region), request_model=request_model, tools=tools, system=system, @@ -14867,7 +14922,7 @@ async def _try_native_responses_count( model_to_use=model_to_use, request=responses_request, policy=policy, - deployment=_with_resolved_credentials(deployment), + deployment=await _with_resolved_credentials(deployment), request_model=request_model, ) ) diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index fd17079b15f..04e6c5d8ba9 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2023,6 +2023,34 @@ ], "default_model_placeholder": "jina_ai/" }, + { + "provider": "KYMA", + "provider_display_name": "Kyma API", + "litellm_provider": "kyma", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://kymaapi.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "kyma/qwen3.7-flash" + }, { "provider": "Laya", "provider_display_name": "Laya", @@ -3584,7 +3612,7 @@ "key": "api_key", "label": "Soniox API Key", "placeholder": null, - "tooltip": "Currently only the async Speech-to-Text REST API (api.soniox.com) is supported. Realtime STT (stt-rt.soniox.com) and TTS (tts-rt.soniox.com) are not yet available.", + "tooltip": "Supports async Speech-to-Text (api.soniox.com) on /v1/audio/transcriptions and realtime Speech-to-Text (stt-rt.soniox.com) on /v1/realtime. TTS (tts-rt.soniox.com) is not yet available.", "required": true, "field_type": "password", "options": null, diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 9f33e0d2adc..b8cdea0dea4 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -367,19 +367,23 @@ async def get_mcp_servers(): tags=["public", "Claude Code Marketplace"], ) async def public_skill_hub(): - """Return enabled (public) Claude Code skills — no auth required.""" + """Return enabled (public) Claude Code skills, or none when the marketplace requires a key.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( MANIFEST_FIELDS, get_installation_preference, - get_prisma_client, ) + from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required + from litellm.proxy.proxy_server import prisma_client from litellm.types.proxy.claude_code_endpoints import ( ListPluginsResponse, PluginListItem, ) + if marketplace_auth_required(): + return ListPluginsResponse(plugins=[], count=0) + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - prisma_client: Final = await get_prisma_client() plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) items: Final = [] for plugin in plugins: @@ -406,7 +410,8 @@ async def public_skill_hub(): ) return ListPluginsResponse(plugins=items, count=len(items)) except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + verbose_logger.exception("Error listing public skills: %s", e) + raise HTTPException(status_code=500, detail="Failed to list skills") @router.get( diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index a7146f13035..2d81dc32181 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -54,6 +54,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( from litellm.proxy.vector_store_endpoints.endpoints import ( build_request_data_from_managed_vector_store, reject_caller_embedding_selection_params, + reject_caller_opensearch_index_name, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -913,6 +914,11 @@ async def rag_query( **retrieval_config, **store_data, } + reject_caller_opensearch_index_name( + caller_body=retrieval_config, + request_fields=merged_retrieval_config, + source="retrieval_config", + ) effective_vector_store_config: Final = ( managed_store_params if managed_store is not None diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9e642a02d49..772bb3e53d2 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -34,6 +34,7 @@ from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.caching.dual_cache import DualCache from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES +from litellm.exceptions import EndpointNotSupportedByModelError from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background @@ -2086,6 +2087,14 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec def _responses_ws_failure_frame(failure: Exception) -> str: + if isinstance(failure, EndpointNotSupportedByModelError): + return json.dumps( + { + "type": "error", + "status": 400, + "error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]}, + } + ) proxy_exception: Final = handle_exception_on_proxy(failure) status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500 error_type: Final = ( @@ -2444,7 +2453,12 @@ async def _responses_websocket_session( with contextlib.suppress(Exception): await websocket.send_text(_responses_ws_failure_frame(e)) with contextlib.suppress(Exception): - await websocket.close(code=1011, reason="Internal server error") + await websocket.close( + code=1008 if isinstance(e, EndpointNotSupportedByModelError) else 1011, + reason="Pre-call error" + if isinstance(e, EndpointNotSupportedByModelError) + else "Internal server error", + ) return crash_row return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index db1951ec9e2..fea4f342b91 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -431,6 +431,28 @@ def raise_if_required_body_param_missing( ) +def endpoint_unsupported_by_model( + route_type: str, + data: Mapping[str, object], + llm_router: LitellmRouter | None, + team_id: str | None, +) -> bool: + model: Final = data.get("model") + if not isinstance(model, str) or llm_router is None: + return False + metadata: Final = data.get("metadata") + request_kwargs: Final = { + **data, + "metadata": { + **(metadata if isinstance(metadata, Mapping) else {}), + "user_api_key_team_id": team_id, + }, + } + return bool(llm_router.get_routable_deployments(model, request_kwargs)) and not llm_router.get_routable_deployments( + model, request_kwargs, call_type=route_type + ) + + class MockTestingParamsDisabledError(HTTPException): def __init__(self, params: tuple[str, ...]): super().__init__( diff --git a/litellm/proxy/shutdown/graceful_shutdown_manager.py b/litellm/proxy/shutdown/graceful_shutdown_manager.py index d7a07ec34e5..9aeee0453ff 100644 --- a/litellm/proxy/shutdown/graceful_shutdown_manager.py +++ b/litellm/proxy/shutdown/graceful_shutdown_manager.py @@ -5,7 +5,8 @@ Kubernetes terminates a pod by sending ``SIGTERM`` and, after ``terminationGracePeriodSeconds``, ``SIGKILL``. On ``SIGTERM`` uvicorn stops accepting connections and waits up to ``GRACEFUL_SHUTDOWN_TIMEOUT`` for open ones, cancelling whatever is still running, before the lifespan teardown that -runs this drain. +runs this drain. The lifespan registers a SIGTERM wrapper before serving +requests so callback draining can subtract the preceding connection drain. A fixed ``preStop`` sleep can not solve this: it has to be sized for the *worst-case* request, so it either wastes time on every routine shutdown or is @@ -20,8 +21,12 @@ granularity of ``InFlightRequestsMiddleware``. import asyncio import os +import signal +import threading import time -from collections.abc import Callable +from collections.abc import Callable, Generator +from contextlib import contextmanager +from types import FrameType from typing import Final from litellm._logging import verbose_proxy_logger @@ -44,6 +49,7 @@ class GracefulShutdownManager: _is_shutting_down: bool = False _shutdown_started_at: float | None = None _drain_performed: bool = False + _sigterm_handler: Callable[[int, FrameType | None], None] | None = None @classmethod def is_shutting_down(cls) -> bool: @@ -71,7 +77,7 @@ class GracefulShutdownManager: return DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT @classmethod - def start_shutdown(cls) -> None: + def start_shutdown(cls, clock: Callable[[], float] = time.monotonic) -> None: """ Mark the worker as draining. Idempotent — repeated calls (e.g. SIGTERM followed by a preStop hit on /health/drain) do not reset the clock. @@ -79,12 +85,42 @@ class GracefulShutdownManager: if cls._is_shutting_down: return cls._is_shutting_down = True - cls._shutdown_started_at = time.monotonic() + cls._shutdown_started_at = clock() verbose_proxy_logger.info( "graceful_shutdown_started in_flight_requests=%s", get_in_flight_requests(), ) + @classmethod + def remaining(cls, cap: float | None = None, *, clock: Callable[[], float] = time.monotonic) -> float: + elapsed: Final = 0.0 if cls._shutdown_started_at is None else clock() - cls._shutdown_started_at + remaining: Final = max(0.0, cls.get_timeout() - elapsed) + return remaining if cap is None else min(max(0.0, cap), remaining) + + @classmethod + @contextmanager + def capture_sigterm(cls, clock: Callable[[], float] = time.monotonic) -> Generator[None, None, None]: + if threading.current_thread() is not threading.main_thread(): + yield + return + previous: Final = signal.getsignal(signal.SIGTERM) + if not callable(previous) or previous is cls._sigterm_handler: + yield + return + + def handle_sigterm(signum: int, frame: FrameType | None) -> None: + cls.start_shutdown(clock) + previous(signum, frame) + + cls._sigterm_handler = handle_sigterm + signal.signal(signal.SIGTERM, handle_sigterm) + try: + yield + finally: + if signal.getsignal(signal.SIGTERM) is handle_sigterm: + signal.signal(signal.SIGTERM, previous) + cls._sigterm_handler = None + @classmethod async def wait_for_drain( cls, @@ -93,13 +129,14 @@ class GracefulShutdownManager: count_fn: Callable[[], int] | None = None, poll_interval: float = _DRAIN_POLL_INTERVAL, log_interval: float = _DRAIN_LOG_INTERVAL, + clock: Callable[[], float] = time.monotonic, ) -> int: """ Poll the in-flight request counter until it reaches the drain target or ``timeout`` seconds elapse. Args: - timeout: Max seconds to wait. Defaults to ``get_timeout()``. + timeout: Max seconds to wait. Defaults to the remaining shutdown budget. exclude_self: When the caller is itself an in-flight HTTP request (the /health/drain endpoint), set this so the caller's own request is not counted as outstanding work. @@ -107,6 +144,7 @@ class GracefulShutdownManager: live ``InFlightRequestsMiddleware`` counter; injectable for tests. poll_interval: Seconds between counter polls. log_interval: Minimum seconds between ``drain_waiting`` log lines. + clock: Monotonic clock shared with the shutdown timestamp. Returns: Number of requests that drained while waiting (>= 0). @@ -120,7 +158,7 @@ class GracefulShutdownManager: cls._drain_performed = True if timeout is None: - timeout = cls.get_timeout() + timeout = cls.remaining(clock=clock) if count_fn is None: count_fn = get_in_flight_requests @@ -128,7 +166,7 @@ class GracefulShutdownManager: # and so counts itself; treat <=1 as "drained" in that case. target: Final = 1 if exclude_self else 0 - start: Final = time.monotonic() + start: Final = clock() initial: Final = count_fn() last_log = start @@ -142,11 +180,11 @@ class GracefulShutdownManager: verbose_proxy_logger.info( "graceful_shutdown_complete drained_requests=%s elapsed_s=%.2f", drained, - time.monotonic() - start, + clock() - start, ) return drained - elapsed = time.monotonic() - start + elapsed = clock() - start if elapsed >= timeout: verbose_proxy_logger.warning( "graceful_shutdown_timeout in_flight_requests=%s elapsed_s=%.2f " @@ -157,7 +195,7 @@ class GracefulShutdownManager: ) return max(0, initial - current) - now = time.monotonic() + now = clock() if now - last_log >= log_interval: verbose_proxy_logger.info( "drain_waiting in_flight_requests=%s elapsed_s=%.2f", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..df9ed611a47 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -227,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -294,6 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -461,7 +464,11 @@ def _sl_attribution_fallback( ) -> str: if standard_logging_payload is None: return "" - return standard_logging_payload.get(field) or "" + return _string_or_empty(standard_logging_payload.get(field)) + + +def _string_or_empty(value: object) -> str: + return value if isinstance(value, str) else "" def _deployment_provider(deployment: DeploymentTypedDict) -> str | None: @@ -679,7 +686,7 @@ def get_logging_payload( _model_id: Final = metadata.get("model_info", {}).get("id", "") or _sl_attribution_fallback( standard_logging_payload, "model_id" ) - _model_group: Final = metadata.get("model_group", "") or _sl_attribution_fallback( + _model_group: Final = _string_or_empty(metadata.get("model_group")) or _sl_attribution_fallback( standard_logging_payload, "model_group" ) _api_base: Final = litellm_params.get("api_base", "") or _sl_attribution_fallback( @@ -738,6 +745,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 1b54dddcf4f..b6a0ebe3c58 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -17,6 +17,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import read_raw_json_body +from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER from litellm.proxy.utils import jsonify_object from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, @@ -24,6 +25,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( get_litellm_managed_vector_store, ) from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository +from litellm.types.utils import LlmProviders from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry @@ -52,6 +54,24 @@ def reject_caller_embedding_selection_params(payload: Mapping[str, object], sour ) +def reject_caller_opensearch_index_name( + caller_body: Mapping[str, object], request_fields: Mapping[str, object], source: str +) -> None: + provider: Final = request_fields.get("custom_llm_provider") + if ( + isinstance(provider, str) + and provider.split("/", 1)[0] == LlmProviders.OPENSEARCH.value + and "opensearch_index_name" in caller_body + ): + raise HTTPException( + status_code=400, + detail={ + "error": f"'opensearch_index_name' cannot be set in {source}. " + "The index comes from the vector store's server-side registration." + }, + ) + + ######################################################## # OpenAI Compatible Endpoints ######################################################## @@ -138,11 +158,17 @@ async def vector_store_search( data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line reject_caller_embedding_selection_params(payload=data, source="the search request body") data["vector_store_id"] = vector_store_id + caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data) # Check for legacy vector store registry (non-managed vector stores) data = await _update_request_data_with_litellm_managed_vector_store_registry( data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict ) + reject_caller_opensearch_index_name( + caller_body=caller_body, + request_fields=JSON_OBJECT_ADAPTER.validate_python(data), + source="the search request body", + ) # The managed_vector_stores pre-call hook will handle: # 1. Decoding managed vector store IDs diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d810de2f24f..987e8be1ed1 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -4,6 +4,6 @@ Supported endpoints: - WebSocket: `/v1/realtime` (with `intent=transcription` for transcription-only sessions) - HTTP: `/v1/realtime/client_secrets`, `/v1/realtime/transcription_sessions` -Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. +Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI, Gemini, Meta, Soniox. For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e9b534bdd89..e03cb44efda 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -34,8 +35,8 @@ from litellm.types.utils import CallTypes, LlmProviders from litellm.utils import ( ModelInfoLookupMiss, ProviderConfigManager, + aresolve_api_key_secret_reference, lookup_model_info_for_registration, - resolve_api_key_secret_reference, ) from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params @@ -55,6 +56,7 @@ from ..llms.openai.realtime.handler import ( confirm_session_started, ) from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async +from ..llms.soniox.realtime.transformation import SonioxRealtimeOptions, soniox_realtime_health_check from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config from ..llms.vertex_ai.vertex_llm_base import VertexBase @@ -75,6 +77,7 @@ vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() _EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) +_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]: @@ -527,6 +530,7 @@ async def _arealtime( custom_llm_provider=_custom_llm_provider, ) + deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs) if live_session_start is not None: await _alive_session( model=model, @@ -537,7 +541,7 @@ async def _arealtime( api_base=dynamic_api_base or litellm_params.api_base, api_key=dynamic_api_key, litellm_metadata=_build_litellm_metadata(kwargs), - litellm_params=kwargs, + litellm_params=deployment_params, ) return @@ -548,6 +552,7 @@ async def _arealtime( model=model, provider=LlmProviders(_custom_llm_provider), drop_params=drop_params, + litellm_params=deployment_params, ) if provider_config is not None: await base_llm_http_handler.async_realtime( @@ -840,7 +845,7 @@ async def realtime_health_check( resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints str | None, api_base or resolved_params.get("api_base") ) - resolved_api_key: Final = resolve_api_key_secret_reference( + resolved_api_key: Final = await aresolve_api_key_secret_reference( cast( # cast-ok: provider parameters expose optional string credentials str | None, api_key or resolved_params.get("api_key") ), @@ -882,6 +887,13 @@ async def realtime_health_check( ) elif custom_llm_provider == "openai": url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model}) + elif custom_llm_provider == "soniox": + return await soniox_realtime_health_check( + model=model, + api_base=resolved_api_base, + api_key=health_check_api_key, + options=SonioxRealtimeOptions.model_validate(dict(resolved_params)), + ) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} diff --git a/litellm/repositories/daily_activity_buckets.py b/litellm/repositories/daily_activity_buckets.py new file mode 100644 index 00000000000..ec6dd0f8c84 --- /dev/null +++ b/litellm/repositories/daily_activity_buckets.py @@ -0,0 +1,39 @@ +from collections.abc import Sequence +from typing import Final + +from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL + +UNKNOWN_MODEL_GROUP: Final = "unknown-model-group" +UNATTRIBUTED_API_KEY: Final = "unattributed" +_RAW_API_KEY_PREFIX: Final = "raw:" + + +def model_bucket(model: str | None) -> str: + return model or UNKNOWN_MODEL_SPEND_LOG_MODEL + + +def model_group_bucket(model_group: str | None, model: str | None) -> str: + return model_group or model or UNKNOWN_MODEL_GROUP + + +def api_key_bucket(api_key: str | None) -> str: + if not api_key: + return UNATTRIBUTED_API_KEY + if api_key == UNATTRIBUTED_API_KEY or api_key.startswith(_RAW_API_KEY_PREFIX): + return _RAW_API_KEY_PREFIX + api_key + return api_key + + +def api_key_from_bucket(api_key: str) -> str: + if api_key == UNATTRIBUTED_API_KEY: + return "" + return api_key.removeprefix(_RAW_API_KEY_PREFIX) + + +def raw_api_key_filter(api_key: str | Sequence[str] | None) -> tuple[str, ...] | None: + """String filters are reporting IDs; sequences are raw keys resolved by authorization.""" + if api_key is None or api_key == "": + return None + if isinstance(api_key, str): + return (api_key_from_bucket(api_key),) + return tuple(api_key) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 4ca23e4e89e..321468c4976 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -10,6 +10,7 @@ from typing_extensions import assert_never from litellm import constants from litellm._logging import verbose_proxy_logger +from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.repositories.chunked_in import find_many_in from litellm.repositories.daily_activity_sql import ( ExportCursor, @@ -344,7 +345,13 @@ class DailyActivityRepository: "date": {"gte": adjusted_start, "lte": adjusted_end}, **({scope.entity_id_field: {"in": list(scope.entity_ids)}} if scope.entity_ids is not None else {}), **exclusion_filter, - **({"model": scope.model} if scope.model else {}), + **( + {"AND": [{"OR": [{"model": None}, {"model": ""}, {"model": scope.model}]}]} + if scope.model == UNKNOWN_MODEL_SPEND_LOG_MODEL + else {"model": scope.model} + if scope.model + else {} + ), **({"api_key": {"in": list(scope.api_keys)}} if scope.api_keys is not None else {}), } count, rows = await asyncio.gather( diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 7ec9fa67a67..4bb52767de6 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -8,11 +8,18 @@ from typing import Final from typing_extensions import assert_never from litellm import constants -from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.constants import PTU_SENTINEL_API_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.repositories.daily_activity_buckets import UNATTRIBUTED_API_KEY, UNKNOWN_MODEL_GROUP from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable, ExportType _API_KEY_ROLLED_UP_BIT: Final = 32 -_MODEL_GROUP_EXPR: Final = "COALESCE(NULLIF(model_group, ''), model)" +_MODEL_EXPR: Final = f"COALESCE(NULLIF(model, ''), '{UNKNOWN_MODEL_SPEND_LOG_MODEL}')" +_MODEL_GROUP_EXPR: Final = f"COALESCE(NULLIF(model_group, ''), NULLIF(model, ''), '{UNKNOWN_MODEL_GROUP}')" + +_API_KEY_EXPR: Final = ( + f"CASE WHEN api_key = '' THEN '{UNATTRIBUTED_API_KEY}' " + f"WHEN api_key = '{UNATTRIBUTED_API_KEY}' OR api_key LIKE 'raw:%' THEN 'raw:' || api_key ELSE api_key END" +) @dataclass(frozen=True, slots=True) @@ -86,7 +93,11 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu if scope.exclude_entity_ids else () ), - *((f"model = ${model_index}",) if scope.model else ()), + *( + (f"{_MODEL_EXPR if scope.model == UNKNOWN_MODEL_SPEND_LOG_MODEL else 'model'} = ${model_index}",) + if scope.model + else () + ), *( ("FALSE",) if scope.api_keys == () @@ -143,7 +154,7 @@ def _top_api_keys_sql(pg_table: str, where_clause: str, *, sentinel_param: int, FROM "{pg_table}" WHERE {where_clause} AND api_key <> ${sentinel_param} GROUP BY api_key - ORDER BY SUM(spend::numeric) DESC, api_key + ORDER BY SUM(spend::numeric) DESC, {_API_KEY_EXPR} LIMIT ${limit_param} """ @@ -162,13 +173,13 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq (SELECT date, NULL::text AS api_key, - model, + {_MODEL_EXPR} AS model, {_MODEL_GROUP_EXPR} AS model_group, custom_llm_provider, mcp_namespaced_tool_name, endpoint, (GROUPING(date) << 6) | {_API_KEY_ROLLED_UP_BIT} - | GROUPING(model, {_MODEL_GROUP_EXPR}, + | GROUPING({_MODEL_EXPR}, {_MODEL_GROUP_EXPR}, custom_llm_provider, mcp_namespaced_tool_name, endpoint) AS group_level, NULL::bigint AS distinct_api_keys,{metric_select} @@ -176,7 +187,7 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq WHERE {where_clause} GROUP BY GROUPING SETS ( (date), - (date, model), + (date, {_MODEL_EXPR}), (date, {_MODEL_GROUP_EXPR}), (date, custom_llm_provider), (date, mcp_namespaced_tool_name), @@ -190,12 +201,12 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq SELECT date, api_key, - model, + {_MODEL_EXPR} AS model, {_MODEL_GROUP_EXPR} AS model_group, custom_llm_provider, mcp_namespaced_tool_name, endpoint, - GROUPING(date, api_key, model, {_MODEL_GROUP_EXPR}, + GROUPING(date, api_key, {_MODEL_EXPR}, {_MODEL_GROUP_EXPR}, custom_llm_provider, mcp_namespaced_tool_name, endpoint) AS group_level, MAX(top_api_keys.distinct_api_keys) AS distinct_api_keys,{metric_select} @@ -203,7 +214,7 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq WHERE {where_clause} GROUP BY GROUPING SETS ( (date, api_key), - (date, model, api_key), + (date, {_MODEL_EXPR}, api_key), (date, {_MODEL_GROUP_EXPR}, api_key), (date, custom_llm_provider, api_key), (date, mcp_namespaced_tool_name, api_key), @@ -313,7 +324,7 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) -> FROM (SELECT 1) AS one LEFT JOIN LATERAL ( SELECT * FROM ranked - ORDER BY rank_spend DESC, api_key + ORDER BY rank_spend DESC, {_API_KEY_EXPR} LIMIT ${limit_param} OFFSET ${offset_param} ) AS page ON TRUE """ @@ -376,7 +387,7 @@ def build_key_search_sql(scope: DailyActivityScope, *, search: str, limit: int) WHERE {where_clause} AND api_key <> ${sentinel_param} AND ( - api_key ILIKE ${search_param} ESCAPE '\\' + {_API_KEY_EXPR} ILIKE ${search_param} ESCAPE '\\' OR api_key IN ( SELECT v.token FROM "LiteLLM_VerificationToken" v LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = v.user_id @@ -392,7 +403,7 @@ def build_key_search_sql(scope: DailyActivityScope, *, search: str, limit: int) ) ) GROUP BY api_key - ORDER BY SUM(spend::numeric) DESC, api_key + ORDER BY SUM(spend::numeric) DESC, {_API_KEY_EXPR} LIMIT ${limit_param} """ return SqlQuery(sql=sql, params=(*where_params, f"%{escaped}%", PTU_SENTINEL_API_KEY, limit)) @@ -406,15 +417,13 @@ def build_model_top_keys_sql( model_param: Final = len(where_params) + 1 sentinel_param: Final = model_param + 1 limit_param: Final = sentinel_param + 1 - model_clause: Final = ( - f"COALESCE(NULLIF(model_group, ''), model) = ${model_param}" if by_model_group else f"model = ${model_param}" - ) + model_clause: Final = f"{_MODEL_GROUP_EXPR if by_model_group else _MODEL_EXPR} = ${model_param}" sql: Final = f""" SELECT api_key,{_key_spend_select()} FROM "{PRISMA_TO_PG_TABLE[scope.table]}" WHERE {where_clause} AND {model_clause} AND api_key <> ${sentinel_param} GROUP BY api_key - ORDER BY SUM(spend::numeric) DESC, api_key + ORDER BY SUM(spend::numeric) DESC, {_API_KEY_EXPR} LIMIT ${limit_param} """ return SqlQuery(sql=sql, params=(*where_params, model_group, PTU_SENTINEL_API_KEY, limit)) @@ -431,7 +440,7 @@ def build_cache_leakage_keys_sql(scope: DailyActivityScope, *, limit: int) -> Sq WHERE {where_clause} AND api_key <> ${sentinel_param} GROUP BY api_key HAVING SUM(prompt_tokens) - SUM(cache_read_input_tokens) > 0 - ORDER BY SUM(prompt_tokens) - SUM(cache_read_input_tokens) DESC, api_key + ORDER BY SUM(prompt_tokens) - SUM(cache_read_input_tokens) DESC, {_API_KEY_EXPR} LIMIT ${limit_param} """ return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, limit)) @@ -483,7 +492,7 @@ def build_export_sql( {alias_expression} AS entity_alias, {output_key} AS api_key, {user_fields}, - {"NULLIF(COALESCE(scoped.model, ''), '')" if export_type is ExportType.DAILY_WITH_MODELS else "NULL::text"} AS model, + {_MODEL_EXPR if export_type is ExportType.DAILY_WITH_MODELS else "NULL::text"} AS model, COALESCE(SUM(scoped.spend), 0)::float AS spend, {flat_cost} AS flat_cost, COALESCE(SUM(scoped.prompt_tokens), 0)::bigint AS prompt_tokens, @@ -521,25 +530,26 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, ) if export_type is ExportType.DAILY_WITH_KEYS: return ( - "scoped.api_key", - "NULLIF(scoped.api_key, '')", - "MAX(COALESCE(vt.key_alias, dvt.key_alias)) AS key_alias, " + _API_KEY_EXPR, + _API_KEY_EXPR, + "MAX(CASE WHEN scoped.api_key = '' THEN 'Unattributed' " + "ELSE COALESCE(vt.key_alias, dvt.key_alias) END) AS key_alias, " "MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, MAX(u.user_email) AS user_email", ( - 'LEFT JOIN "LiteLLM_VerificationToken" vt ON vt.token = scoped.api_key', + "LEFT JOIN \"LiteLLM_VerificationToken\" vt ON vt.token = scoped.api_key AND scoped.api_key <> ''", """LEFT JOIN LATERAL ( SELECT key_alias, user_id FROM "LiteLLM_DeletedVerificationToken" WHERE token = scoped.api_key ORDER BY deleted_at DESC LIMIT 1 - ) dvt ON vt.token IS NULL""", + ) dvt ON vt.token IS NULL AND scoped.api_key <> ''""", 'LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = COALESCE(vt.user_id, dvt.user_id)', ), ) if export_type is ExportType.DAILY_WITH_MODELS: return ( - "COALESCE(scoped.model, '')", + _MODEL_EXPR, "NULL::text", "NULL::text AS key_alias, NULL::text AS user_id, NULL::text AS user_email", (), diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 65d9b89cf79..e344ce46be7 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -19,6 +19,12 @@ from litellm.types.rerank import RerankResponse from litellm.types.router import * from litellm.utils import ProviderConfigManager, client, exception_type +_PROVIDERS_RESOLVING_THEIR_OWN_RERANK_BASE: Final = ( + litellm.LlmProviders.DASHSCOPE, + litellm.LlmProviders.QWENCLOUD, + litellm.LlmProviders.QWEN_AI_PLATFORM, +) + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here together_rerank: Final = TogetherAIRerank() @@ -543,7 +549,11 @@ def rerank( logging_obj=litellm_logging_obj, timeout=optional_params.timeout, api_key=dynamic_api_key or optional_params.api_key, - api_base=dynamic_api_base or optional_params.api_base, + api_base=( + optional_params.api_base or dynamic_api_base + if _custom_llm_provider in _PROVIDERS_RESOLVING_THEIR_OWN_RERANK_BASE + else dynamic_api_base or optional_params.api_base + ), _is_async=_is_async, headers=headers or litellm.headers or {}, client=client, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5bed6037fa4..98356e9a5d4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -90,9 +90,9 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, + AgenticLoopParams, CallTypes, CostPerToken, - CustomPricingLiteLLMParams, all_litellm_params, ) from litellm.utils import ( @@ -565,12 +565,6 @@ async def aresponses_api_with_mcp( return response -def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: - return MappingProxyType( - litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) - ) - - class _DeploymentModelInfo(BaseModel): model_config = ConfigDict(frozen=True) id: object = None @@ -1580,6 +1574,7 @@ def responses( local_vars=local_vars, use_chat_completions_api=use_chat_completions_api, ) + litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) from litellm.litellm_core_utils.core_helpers import attribute_or_none @@ -1798,7 +1793,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, - **_per_call_custom_pricing(litellm_params), + **litellm_params.custom_pricing_overrides(), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8959c32fff2..6a82304b76a 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -227,7 +227,7 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None: return _, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout) if pending: - verbose_logger.warning("Timed out draining %d logging tasks on shutdown, dropping them", len(pending)) + verbose_logger.warning("Timed out draining logging tasks on shutdown: %d tasks still alive", len(pending)) _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..b0db0516658 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -131,14 +131,14 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts @@ -287,7 +287,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. diff --git a/litellm/router.py b/litellm/router.py index 3809a5d6555..7d566bd0744 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -75,7 +75,7 @@ from typing_extensions import assert_never, overload import litellm import litellm.litellm_core_utils.exception_mapping_utils from litellm import get_secret_str -from litellm._internal_context import service_target, with_service_target +from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid from litellm.caching.caching import ( @@ -287,6 +287,11 @@ from litellm.router_utils.embedding_batch_size_check import ( embedding_deployment_selection, log_rejection_before_provider_call, ) +from litellm.router_utils.endpoint_eligibility import ( + eligible_deployments, + endpoint_selection, + warn_on_unenforced_endpoint_flag, +) from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, AttemptedFallbackTargets, @@ -482,6 +487,7 @@ from litellm.utils import ( ProviderConfigManager, Rules, TextCompletionResponse, + aresolve_api_key_secret_reference, function_setup, get_llm_provider, get_non_default_completion_params, @@ -492,7 +498,6 @@ from litellm.utils import ( lookup_model_info_for_registration, model_info_with_pricing_overrides, provider_rejectable_params, - resolve_api_key_secret_reference, set_live_deployment_replay, token_counter, ) @@ -754,7 +759,7 @@ def _every_deployment_refused( def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: - if silent_model is None: + if silent_model is None or in_mirrored_call(): return () try: targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) @@ -1449,7 +1454,7 @@ class Router: self.assistants_config = assistants_config self.search_tools = search_tools or [] self.guardrail_list = guardrail_list or [] - self.deployment_names: list = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 + self.deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 self.deployment_latency_map = {} ### CACHING ### cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache @@ -3372,16 +3377,17 @@ class Router: remaining_kwargs: Final = MappingProxyType( {key: value for key, value in silent_kwargs.items() if key != "stream"} ) - response: Final = await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - stream=bool(silent_kwargs.get("stream", False)), - **remaining_kwargs, - ) - if not isinstance(response, CustomStreamWrapper): - return - async for _ in response: - pass + with mirrored_call(): + response: Final = await self.acompletion( + model=silent_model, + messages=cast(list[AllMessageValues], messages), + stream=bool(silent_kwargs.get("stream", False)), + **remaining_kwargs, + ) + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): """ @@ -6053,12 +6059,13 @@ class Router: kwargs.setdefault("metadata", {}).update({"model_group": model}) # pick the one that is available (lowest TPM/RPM) - deployment: Final = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with endpoint_selection("text_completion", model=model): + deployment: Final = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) data: Final = deployment["litellm_params"].copy() for k, v in self.default_litellm_params.items(): @@ -6914,6 +6921,7 @@ class Router: """ model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group retry_kwargs: Final = mid_stream_retry_kwargs(initial_kwargs) + handler: Final = retry_kwargs.get("original_generic_function") healthy_deployments, all_deployments = await self._async_get_healthy_deployments( model=model_group, parent_otel_span=get_parent_otel_span_from_kwargs(retry_kwargs), @@ -6944,7 +6952,10 @@ class Router: ) carry_superseded_attempt_usage(retry_kwargs) try: - response = await attempt_function(**retry_kwargs) + with endpoint_selection( + getattr(handler, "__name__", None), getattr(handler, "__module__", None), model=model_group + ): + response = await attempt_function(**retry_kwargs) except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry wrapped = _stream_fallback_error_for_raised(retry_error, model_group, False, e.llm_provider) if wrapped is None: @@ -9314,9 +9325,18 @@ class Router: Handler for making a call to the .completion()/.embeddings()/etc. functions. """ model_group: Final = kwargs.get("model") - response = original_function(*args, **kwargs) - if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): - response = await response + generic_function: Final = _OPAQUE_VALUE_ADAPTER.validate_python(kwargs.get("original_generic_function")) + endpoint_function: Final = ( + generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function) + ) + with endpoint_selection( + getattr(endpoint_function, "__name__", None), + getattr(endpoint_function, "__module__", None), + model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group), + ): + response = original_function(*args, **kwargs) + if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): + response = await response await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) ## PROCESS RESPONSE HEADERS response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs) @@ -10637,6 +10657,7 @@ class Router: declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params) warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag(model_name=_model_name, model_info=_model_info) deployment = without_model_capability_flags( Deployment( **deployment_info, @@ -11440,6 +11461,9 @@ class Router: declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag( + model_name=deployment.model_name, model_info=deployment.model_info.model_dump() + ) served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) # add to model list @@ -12055,7 +12079,7 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return - api_key: Final = resolve_api_key_secret_reference( + api_key: Final = await aresolve_api_key_secret_reference( params.api_key or dynamic_api_key, model=model, llm_provider=provider, @@ -14796,9 +14820,15 @@ class Router: """ return [m for m in self.model_list if m["litellm_params"]["model"] == model] - def get_routable_deployments(self, model: str) -> tuple[DeploymentTypedDict, ...]: + def get_routable_deployments( + self, + model: str, + request_kwargs: Mapping[str, object] = MappingProxyType({}), + call_type: str | None = None, + ) -> tuple[DeploymentTypedDict, ...]: try: - _, candidates = self._common_checks_available_deployment(model=model, request_kwargs={}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped + with endpoint_selection(call_type, model=model): + _, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped except (litellm.BadRequestError, ValueError): return () if isinstance(candidates, dict): @@ -15074,7 +15104,10 @@ class Router: model=model, llm_provider="", ) - return selectable + eligible: Final = eligible_deployments(model, selectable) + if isinstance(eligible, tuple): + return list(eligible) + raise eligible def _filter_reserved_deployments( self, diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py new file mode 100644 index 00000000000..4a87e582c97 --- /dev/null +++ b/litellm/router_utils/endpoint_eligibility.py @@ -0,0 +1,120 @@ +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from types import MappingProxyType +from typing import Final, TypeVar + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator + +from litellm._logging import verbose_router_logger +from litellm.exceptions import EndpointNotSupportedByModelError + +CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType( + { + "completion": "/v1/chat/completions", + "acompletion": "/v1/chat/completions", + "_completion": "/v1/chat/completions", + "_acompletion": "/v1/chat/completions", + "text_completion": "/v1/completions", + "atext_completion": "/v1/completions", + "_atext_completion": "/v1/completions", + "anthropic_messages": "/v1/messages", + "aresponses": "/v1/responses", + "agenerate_content": "generateContent", + "agenerate_content_stream": "streamGenerateContent", + "acreate_interaction": "/v1beta/interactions", + "litellm.interactions.main.acreate": "/v1beta/interactions", + "litellm.interactions.main.create": "/v1beta/interactions", + "_aresponses_websocket": "/v1/responses", + "acompact_responses": "/v1/responses/compact", + } +) + + +def _endpoint_path(endpoint: str) -> str: + rooted: Final = "/" + endpoint.strip().strip("/").casefold() + path: Final = rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") + return "/" + path.rsplit(":", 1)[-1] if ":" in path else path + + +CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values()) +_Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object]) +_ENFORCE_FLAG: Final = TypeAdapter(bool | None) +_REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar( + "router_request_endpoint", default=(None, None) +) + + +class _EndpointDeclaration(BaseModel): + model_config = ConfigDict(frozen=True) + + enforce_supported_endpoints: bool | None = None + supported_endpoints: tuple[str, ...] | None = None + + @field_validator("supported_endpoints", mode="before") + @classmethod + def _one_endpoint_as_a_list(cls, value: object) -> object: + return (value,) if isinstance(value, str) else value + + +@contextmanager +def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]: + qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type + endpoint: Final = CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None + token: Final = _REQUEST_SELECTION.set((endpoint, model if isinstance(model, str) else None)) + try: + yield + finally: + _REQUEST_SELECTION.reset(token) + + +def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None: + try: + declaration: Final = _EndpointDeclaration.model_validate(model_info) + except ValidationError: + return None + return declaration.supported_endpoints if declaration.enforce_supported_endpoints else None + + +def warn_on_unenforced_endpoint_flag(model_name: str, model_info: Mapping[str, object]) -> str | None: + try: + flagged: Final = _ENFORCE_FLAG.validate_python(model_info.get("enforce_supported_endpoints")) + except ValidationError: + return None + if not flagged or _enforced_endpoints(model_info) is not None: + return None + warning: Final = ( + f"Deployment '{model_name}' sets enforce_supported_endpoints without a supported_endpoints list of " + 'strings, so litellm enforces nothing for it. Set supported_endpoints, e.g. ["/v1/embeddings"].' + ) + verbose_router_logger.warning(warning) + return warning + + +def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None: + declared: Final = _enforced_endpoints(deployment.get("model_info")) + if declared is None: + return None + endpoints: Final = frozenset(declared) + normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints) + return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None + + +def eligible_deployments( + model: str, deployments: Sequence[_Deployment] +) -> tuple[_Deployment, ...] | EndpointNotSupportedByModelError: + endpoint, requested_model = _REQUEST_SELECTION.get() + if endpoint is None or not deployments: + return tuple(deployments) + eligible: Final = tuple(deployment for deployment in deployments if non_chat_endpoints(deployment) is None) + if eligible: + return eligible + requested: Final = requested_model or model + supported: Final = ", ".join( + sorted(frozenset[str]().union(*(non_chat_endpoints(deployment) or frozenset() for deployment in deployments))) + ) + return EndpointNotSupportedByModelError( + message=f"{endpoint} is not supported by model={requested}. Supported endpoints: {supported or 'none'}", + model=model, + llm_provider="", + ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..3c3fad93579 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,6 +426,27 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, @@ -434,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 7364a1d72f2..62b48621c05 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -14,6 +14,7 @@ class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... def trace_encode_error(message: str) -> bytes: ... +def bundled_model_cost_map() -> bytes: ... @final class NativeClickHouseSpendConfig: @@ -220,6 +221,7 @@ __all__ = [ "aocr", "aresponses", "atranscription", + "bundled_model_cost_map", "completion", "embedding", "gil_stats", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74cd28161a4..fe52f8416d0 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -96,6 +96,8 @@ class Feature(str, Enum): """litellm.include_cost_in_streaming_usage, where Python adds cost to streamed usage.""" THINKING_SIGNATURE_RETRY = "thinking_signature_retry" """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" + TOOL_CUSTOM_FIELD = "tool_custom_field" + """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -176,6 +178,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.CACHE_CONTROL_INJECTION, Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, + Feature.TOOL_CUSTOM_FIELD, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 5f75858d350..52917126b3b 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -43,6 +43,11 @@ def thinking_signature_retry(request: NativeCall) -> bool: return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ()) +def tool_custom_field(request: NativeCall) -> bool: + tools: Final = optional_sequence(request.resolved.get("tools")) or () + return any("custom" in (optional_mapping(tool) or {}) for tool in tools) + + def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 588bdb1409c..02cf53d00ec 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -5,7 +5,7 @@ import time import traceback from collections.abc import Callable from dataclasses import dataclass -from typing import Final +from typing import Final, TypeAlias import httpx from pydantic import TypeAdapter, ValidationError @@ -13,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger from litellm.caching.caching import DualCache +from litellm.litellm_core_utils.env_utils import get_env_float from litellm.litellm_core_utils.single_flight import single_flight_lru_cache from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.secret_managers.get_azure_ad_token_provider import ( @@ -167,12 +168,37 @@ def get_secret_str( LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/") +GOOGLE_OIDC_SECRET_PREFIX: Final = "oidc/google/" +API_KEY_OIDC_SECRET_PREFIXES: Final = (*LOCAL_OIDC_SECRET_PREFIXES, GOOGLE_OIDC_SECRET_PREFIX) +_GOOGLE_METADATA_DEFAULT_HOST: Final = "metadata.google.internal" +_GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS: Final = 3.0 +_GOOGLE_TOKEN_UNAVAILABLE: Final = "the metadata server did not return an ID token" + + +def _google_metadata_identity_url() -> str: + host: Final = os.getenv("GCE_METADATA_HOST") or _GOOGLE_METADATA_DEFAULT_HOST + return f"http://{host}/computeMetadata/v1/instance/service-accounts/default/identity" + + +def _google_metadata_timeout() -> float: + seconds: Final = get_env_float("GCE_METADATA_TIMEOUT", _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS) + return seconds if seconds > 0 else _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS + + +class GoogleMetadataStatusError(ValueError): + def __init__(self, status_code: int) -> None: + super().__init__(f"Google OIDC provider failed with HTTP {status_code}") + self.status_code: Final = status_code @dataclass(frozen=True, slots=True) class UnresolvableSecretReference: kind: str reason: str + retryable: bool = False + + +OidcResolver: TypeAlias = Callable[[str], "str | UnresolvableSecretReference"] _OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"}) @@ -198,11 +224,86 @@ def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReferenc return secret -def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference: +def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretReference: + if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX): + return resolve_local_oidc_reference(value) + try: + token: Final = _google_id_token( + value, value.removeprefix(GOOGLE_OIDC_SECRET_PREFIX), timeout=_google_metadata_timeout() + ) + except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: + return _logged_unresolvable( + value, + "oidc/google", + _GOOGLE_TOKEN_UNAVAILABLE, + f"{type(exc).__name__}: {exc}", + retryable=_metadata_server_failed_to_answer(exc), + ) + if not token: + return _logged_unresolvable(value, "oidc/google", "it resolved to an empty value") + return token + + +def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool: + match exc: + case httpx.TransportError(): + return True + case GoogleMetadataStatusError(status_code=status_code): + return status_code >= 500 + case _: + return False + + +def _google_id_token(reference: str, audience: str, timeout: float | None) -> str: + cached: Final = cached_oidc_token(reference) + if cached is not None: + return cached + oidc_client: Final = _get_oidc_http_handler() + try: + # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature + response: Final = oidc_client.get( + _google_metadata_identity_url(), + params={"audience": audience}, + headers={"Metadata-Flavor": "Google"}, + timeout=timeout, + ) + finally: + oidc_client.close() + if response.status_code != 200: + raise GoogleMetadataStatusError(response.status_code) + token: Final = response.text + ttl: Final = _oidc_token_cache_ttl(token, 3600 - 60) + if not token: + verbose_logger.warning("Google OIDC token for %s is empty; not caching it", _single_line(reference)) + elif ttl > 0: + oidc_cache.set_cache(key=reference, value=token, ttl=ttl) + else: + verbose_logger.warning( + "Google OIDC token for %s is already expired or expires within %ss; not caching it", + _single_line(reference), + _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS, + ) + return token + + +def cached_oidc_token(reference: str) -> str | None: + cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped + return cached if isinstance(cached, str) and cached else None + + +def _logged_unresolvable( + reference: str, kind: str, reason: str, detail: str | None = None, retryable: bool = False +) -> UnresolvableSecretReference: verbose_logger.warning( - "Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason + "Secret reference %s could not be resolved: %s", + _single_line(reference), + reason if detail is None else f"{reason} ({_single_line(detail)})", ) - return UnresolvableSecretReference(kind=kind, reason=reason) + return UnresolvableSecretReference(kind=kind, reason=reason, retryable=retryable) + + +def _single_line(text: str) -> str: + return text.replace("\r", "").replace("\n", "") def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str: @@ -273,31 +374,7 @@ def get_secret( oidc_aud = "/".join(secret_name_split.split("/")[1:]) # TODO: Add caching for HTTP requests if oidc_provider == "google": - oidc_token = oidc_cache.get_cache(key=secret_name) - if oidc_token is not None: - return oidc_token - - oidc_client = _get_oidc_http_handler() - # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature - response = oidc_client.get( - "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity", - params={"audience": oidc_aud}, - headers={"Metadata-Flavor": "Google"}, - ) - if response.status_code == 200: - oidc_token = response.text - ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60) - if ttl > 0: - oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl) - else: - verbose_logger.warning( - "Google OIDC token for %s is already expired or expires within %ss; not caching it", - secret_name, - _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS, - ) - return oidc_token - else: - raise ValueError("Google OIDC provider failed") + return _google_id_token(secret_name, oidc_aud, timeout=None) elif oidc_provider == "circleci": # https://circleci.com/docs/openid-connect-tokens/ env_secret = os.getenv("CIRCLE_OIDC_TOKEN") diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 13542f22f88..77e75dea030 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -97,9 +97,9 @@ class AgenticLoopRequestPatch(LiteLLMBaseModel): model: str | None = None messages: list[dict[str, Any]] | None = None - tools: list[dict[str, Any]] | None = None + tools: list[dict[str, object]] | None = None max_tokens: int | None = None - optional_params: dict[str, Any] = Field(default_factory=dict) + optional_params: dict[str, object] = Field(default_factory=dict) kwargs: dict[str, Any] = Field(default_factory=dict) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index f4fcabf53ed..f2042cb7967 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -5,7 +5,7 @@ API Reference: https://docs.datadoghq.com/llm_observability/setup/api/?tab=examp """ from collections.abc import Sequence -from typing import Any, Literal +from typing import Literal from typing_extensions import ReadOnly, TypedDict @@ -16,7 +16,7 @@ class ToolCall(TypedDict, total=False): """A tool call on a message, as LLM Obs names its fields.""" name: ReadOnly[str] - arguments: ReadOnly[dict[str, Any] | str] # parsed object, or the raw string when it will not parse to one + arguments: ReadOnly[dict[str, object] | str] # parsed object, or the raw string when it will not parse to one tool_id: ReadOnly[str] type: ReadOnly[str] @@ -35,7 +35,7 @@ class ToolDefinition(TypedDict, total=False): name: ReadOnly[str] description: ReadOnly[str] - schema: ReadOnly[dict[str, Any]] + schema: ReadOnly[dict[str, object]] class Message(TypedDict, total=False): @@ -49,13 +49,15 @@ class Message(TypedDict, total=False): class InputMeta(TypedDict): - messages: Sequence[ - Message | dict[str, Any] # changed to fit with tool calls + messages: ReadOnly[ + Sequence[ + Message | dict[str, object] # changed to fit with tool calls + ] ] # Relevant Issue: https://github.com/BerriAI/litellm/issues/9494 class OutputMeta(TypedDict): - messages: Sequence[Any] + messages: ReadOnly[Sequence[object]] class DDLLMObsError(TypedDict, total=False): @@ -71,7 +73,7 @@ class Meta(TypedDict, total=False): kind: Literal["llm", "tool", "task", "embedding", "retrieval"] input: InputMeta # The span's input information. output: OutputMeta # The span's output information. - metadata: dict[str, Any] + metadata: ReadOnly[dict[str, object]] error: DDLLMObsError | None # Error information on the span tool_definitions: ReadOnly[Sequence[ToolDefinition]] # The tools offered to the model on this request diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index de83e6ea66e..50c4a2340a3 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -832,6 +832,7 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" +ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01" ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01" ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 6ce51abcd8a..d9d86cf54b3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1600,7 +1600,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): @field_validator("reasoning", mode="before") @classmethod - def validate_reasoning_to_dict(cls, value: Any) -> dict[str, Any] | None: + def validate_reasoning_to_dict(cls, value: Any) -> builtins.object: """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" if value is None: return None diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index f1176f777c7..a938f6a7b1c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from enum import Enum -from typing import Any, Final, Literal, Protocol +from typing import Final, Literal, Protocol from typing_extensions import ( ReadOnly, @@ -140,7 +140,7 @@ class Schema(TypedDict, total=False): title: str description: str nullable: bool - default: Any + default: ReadOnly[object] items: "Schema" minItems: str maxItems: str @@ -155,7 +155,7 @@ class Schema(TypedDict, total=False): minLength: str maxLength: str pattern: str - example: Any + example: ReadOnly[object] anyOf: list["Schema"] @@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False): class CachedContentRequestBody(TypedDict, total=False): contents: Required[list[ContentType]] system_instruction: SystemInstructions - tools: Tools + tools: Tools | None toolConfig: ToolConfig model: Required[str] # Format: models/{model} ttl: str # ending in 's' - Example: "3.5s". @@ -794,9 +794,9 @@ class VertexVideoGenerationResponse(TypedDict, total=False): name: str done: bool - response: dict[str, Any] - metadata: dict[str, Any] - error: dict[str, Any] + response: ReadOnly[dict[str, object]] + metadata: ReadOnly[dict[str, object]] + error: ReadOnly[dict[str, object]] VERTEX_CREDENTIALS_TYPES = str | dict[str, str] diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index d396c79b844..ca6ada0d1ec 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -15,4 +15,5 @@ class UiDiscoveryEndpoints(LiteLLMBaseModel): is_control_plane: bool = False mcp_stdio_enabled: bool = False use_si_terminology: bool = False + claude_code_marketplace_auth_required: bool = False workers: list[WorkerRegistryEntry] = Field(default=[]) diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index d38e54986b9..4e4983c53f0 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -48,7 +48,7 @@ class GenericGuardrailAPIMetadata(TypedDict, total=False): class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel): """Optional parameters for the Generic Guardrail API""" - additional_provider_specific_params: dict[str, Any] | None = Field( + additional_provider_specific_params: dict[str, object] | None = Field( default=None, description="Additional provider-specific parameters to send with the guardrail request", ) @@ -161,7 +161,7 @@ class GenericGuardrailAPIRequest(LiteLLMBaseModel): default=None, description="LiteLLM library version running this proxy.", ) - additional_provider_specific_params: dict[str, Any] | None = None + additional_provider_specific_params: dict[str, object] | None = None tool_calls: list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] | None = None model: str | None = None # the model being used for the LLM call diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py index b58d1bff8a2..b04be71dc9c 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py @@ -1,7 +1,5 @@ -from typing import Any - from pydantic import Field -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -14,41 +12,41 @@ from .base import IBMGuardrailsBaseConfigModel class IBMDetectorRequestBodyDetectorServer(TypedDict): """Request body for calling IBM Detector Server directly""" - contents: list[str] - detector_params: dict[str, Any] + contents: ReadOnly[list[str]] + detector_params: ReadOnly[dict[str, object]] class IBMDetectorRequestBodyOrchestrator(TypedDict): """Request body for calling IBM Detector via FMS Guardrails Orchestrator""" - content: str - detectors: dict[str, dict[str, Any]] + content: ReadOnly[str] + detectors: ReadOnly[dict[str, dict[str, object]]] class IBMDetectorDetection(TypedDict, total=False): """Individual detection from IBM Detector""" - start: int - end: int - text: str - detection: str - detection_type: str - score: float - evidences: list[Any] - metadata: dict[str, Any] - detector_id: str | None # Only present in orchestrator response + start: ReadOnly[int] + end: ReadOnly[int] + text: ReadOnly[str] + detection: ReadOnly[str] + detection_type: ReadOnly[str] + score: ReadOnly[float] + evidences: ReadOnly[list[object]] + metadata: ReadOnly[dict[str, object]] + detector_id: ReadOnly[str | None] # Only present in orchestrator response class IBMDetectorResponseDetectorServer(TypedDict): """Response from IBM Detector Server (returns list of lists)""" - detections: list[list[IBMDetectorDetection]] + detections: ReadOnly[list[list[IBMDetectorDetection]]] class IBMDetectorResponseOrchestrator(TypedDict): """Response from IBM FMS Guardrails Orchestrator""" - detections: list[IBMDetectorDetection] + detections: ReadOnly[list[IBMDetectorDetection]] # Pydantic Config Models @@ -57,12 +55,12 @@ class IBMDetectorResponseOrchestrator(TypedDict): class IBMDetectorOptionalParams(LiteLLMBaseModel): """Optional parameters for IBM Detector guardrail""" - detector_params: dict[str, Any] | None = Field( + detector_params: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of arguments to pass to the detector.", ) - extra_headers: dict[str, Any] | None = Field( + extra_headers: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of extra headers to pass to the detector.", ) diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index f4348391710..d4f41b538e6 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict @@ -50,14 +50,14 @@ class SuccessfulKeyUpdate(LiteLLMBaseModel): """Successfully updated key with its updated information""" key: str - key_info: dict[str, Any] + key_info: dict[str, object] class FailedKeyUpdate(LiteLLMBaseModel): """Failed key update with reason""" key: str - key_info: dict[str, Any] | None = None + key_info: dict[str, object] | None = None failed_reason: str @@ -78,14 +78,14 @@ class KeyUpdateFields(LiteLLMBaseModel): max_budget: float | None = None budget_id: str | None = None budget_duration: str | None = None - budget_limits: list[Any] | None = None - model_max_budget: dict[str, Any] | None = None + budget_limits: list[object] | None = None + model_max_budget: dict[str, object] | None = None # Rate limits tpm_limit: int | None = None rpm_limit: int | None = None - model_tpm_limit: dict[str, Any] | None = None - model_rpm_limit: dict[str, Any] | None = None + model_tpm_limit: dict[str, object] | None = None + model_rpm_limit: dict[str, object] | None = None max_parallel_requests: int | None = None rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None @@ -99,7 +99,7 @@ class KeyUpdateFields(LiteLLMBaseModel): # Operational metadata tags: list[str] | None = None - metadata: dict[str, Any] | None = None + metadata: dict[str, object] | None = None @model_validator(mode="after") def validate_temp_budget(self) -> "KeyUpdateFields": diff --git a/litellm/types/router.py b/litellm/types/router.py index 60c9e596588..56dcad4051a 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -67,6 +67,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" @@ -266,6 +270,7 @@ def _whole_float_as_int(value: object) -> object: class ModelInfo(MirroredPricingParams): + enforce_supported_endpoints: bool | None = None id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config. updated_at: datetime.datetime | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 266a6a20131..3bdcba75778 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -263,10 +263,10 @@ class AgenticLoopParams(TypedDict, total=False): """ model: str - """The model string with provider prefix (e.g., 'bedrock/invoke/...')""" + """The model as the caller requested it, usually with its provider prefix (e.g., 'bedrock/invoke/...')""" custom_llm_provider: str - """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + """The LLM provider name (e.g., 'bedrock', 'anthropic'), when the recorder resolved one""" class OffPeakWindow(TypedDict, total=False): @@ -3374,6 +3374,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): @@ -4130,11 +4131,28 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): """ return {k: v for k, v in model_info.items() if k not in cls.model_fields} + def custom_pricing_overrides(self) -> Mapping[str, object]: + return MappingProxyType( + self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + def per_second_pricing(self) -> Mapping[str, float]: + return MappingProxyType( + self.model_dump( + include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True + ) + ) + DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( - {"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"} + { + "supports_regex_lookaround", + "supports_assistant_prefill", + "supports_bedrock_native_structured_output", + "supports_pdf_input", + } ) DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"}) @@ -4546,6 +4564,7 @@ class LlmProviders(str, Enum): S3_VECTORS = "s3_vectors" VALKEY = "valkey" MONGODB = "mongodb" + OPENSEARCH = "opensearch" HELICONE = "helicone" HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" @@ -4599,6 +4618,7 @@ class LlmProviders(str, Enum): CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + KYMA = "kyma" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 26ce38d63b3..4a0e4cafb22 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -28,9 +28,10 @@ import textwrap import threading import time import traceback +from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext -from dataclasses import dataclass, field -from functools import lru_cache, wraps +from dataclasses import dataclass, field, replace +from functools import lru_cache, partial, wraps from importlib import resources from inspect import iscoroutine from io import StringIO @@ -91,7 +92,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -220,6 +221,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -476,11 +478,14 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig from litellm.secret_managers.main import ( - LOCAL_OIDC_SECRET_PREFIXES, + API_KEY_OIDC_SECRET_PREFIXES, + GOOGLE_OIDC_SECRET_PREFIX, + OidcResolver, UnresolvableSecretReference, + cached_oidc_token, get_secret, oidc_reference_kind, - resolve_local_oidc_reference, + resolve_api_key_oidc_reference, ) from ._logging import is_debugging_on, verbose_logger @@ -783,27 +788,97 @@ def _named_credential_values(kwargs: Mapping[str, object]) -> Mapping[str, objec ) -def resolve_api_key_secret_reference( +async def aresolve_api_key_secret_reference( api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] ) -> str | None: if not isinstance(api_key, AdminApiKeySecretReference): return api_key - return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream) + return await _resolved_off_the_event_loop( + partial(_secret_behind_api_key_reference, api_key, model, llm_provider, upstream), api_key + ) + + +_OIDC_TOKEN_FETCH_EXECUTOR: Final = ThreadPoolExecutor(thread_name_prefix="litellm-oidc-token-fetch") +_PASS_THROUGH_CALL_TYPES: Final = frozenset( + {CallTypes.allm_passthrough_route.value, CallTypes.llm_passthrough_route.value} +) +_Resolved: Final = TypeVar("_Resolved") + + +async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolved], api_key: object) -> _Resolved: + if not isinstance(api_key, AdminApiKeySecretReference) or not api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX): + return resolve(resolve_api_key_oidc_reference) + match cached_oidc_token(api_key): + case str() as token: + return resolve(lambda _reference: token) + case None: + _raise_if_the_reference_may_not_be_used(resolve) + fetched: Final = await asyncio.shield( + _google_token_fetch(asyncio.get_running_loop(), api_key).start_or_join() + ) + return resolve(lambda _reference: fetched) + + +def _raise_if_the_reference_may_not_be_used(resolve: Callable[[OidcResolver], object]) -> None: + _ = resolve(lambda _reference: "") + + +@dataclass(slots=True) +class _GoogleTokenFetch: + reference: str + in_flight: asyncio.Future[str | UnresolvableSecretReference] | None = None + + def start_or_join(self) -> asyncio.Future[str | UnresolvableSecretReference]: + if self.in_flight is not None: + return self.in_flight + caller_context: Final = contextvars.copy_context() + fetch: Final = asyncio.get_running_loop().run_in_executor( + _OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve_api_key_oidc_reference, self.reference) + ) + fetch.add_done_callback(self._land) + self.in_flight = fetch + return fetch + + def _land(self, _fetch: asyncio.Future[str | UnresolvableSecretReference]) -> None: + self.in_flight = None + + +@lru_cache(maxsize=256) +def _google_token_fetch(loop: asyncio.AbstractEventLoop, reference: str) -> _GoogleTokenFetch: + return _GoogleTokenFetch(reference) def _secret_behind_api_key_reference( - reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object] + reference: AdminApiKeySecretReference, + model: object, + llm_provider: object, + upstream: Mapping[str, object], + resolve_oidc: OidcResolver = resolve_api_key_oidc_reference, + call_type: str | None = None, ) -> str: if not reference.may_reach(upstream): raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model) - if not reference.startswith(LOCAL_OIDC_SECRET_PREFIXES): + if not reference.startswith(API_KEY_OIDC_SECRET_PREFIXES): raise _secret_reference_error( f"api_key is an {oidc_reference_kind(reference)} secret reference, a kind api_key does not support. " - "Use oidc/file/ or oidc/env/.", + "Use oidc/file/, oidc/env/ or oidc/google/.", llm_provider=llm_provider, model=model, ) - match resolve_local_oidc_reference(reference): + if call_type in _PASS_THROUGH_CALL_TYPES and reference.startswith(GOOGLE_OIDC_SECRET_PREFIX): + raise _secret_reference_error( + "api_key oidc/google secret reference could not be resolved: pass-through routes do not fetch Google ID " + "tokens. Use a static key, oidc/file/ or oidc/env/.", + llm_provider=llm_provider, + model=model, + ) + match resolve_oidc(reference): + case UnresolvableSecretReference(kind=kind, reason=reason, retryable=True): + raise ServiceUnavailableError( + message=f"api_key {kind} secret reference could not be resolved: {reason}", + llm_provider=llm_provider, + model=model, + ) case UnresolvableSecretReference(kind=kind, reason=reason): raise _secret_reference_error( f"api_key {kind} secret reference could not be resolved: {reason}", @@ -835,21 +910,39 @@ def with_named_credential(kwargs: Mapping[str, object]) -> Mapping[str, object]: return {**kwargs, **_named_credential_values(kwargs)} -def load_credentials_from_list(kwargs: dict[str, object]) -> None: +def load_credentials_from_list(kwargs: dict[str, object], call_type: str | None = None) -> None: """ Updates kwargs with the credentials if credential_name in kwarg, then swaps an admin-configured oidc/ api_key reference for the secret it points to, read now so a rotated token is used on the next call """ - kwargs.update(_credential_updates(kwargs)) + kwargs.update(_credential_updates(kwargs, _named_credential_values(kwargs), call_type=call_type)) -def _credential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]: +async def acredential_updates(kwargs: Mapping[str, object], call_type: str | None = None) -> Mapping[str, object]: credential_values: Final = _named_credential_values(kwargs) - api_key: Final = kwargs.get("api_key", credential_values.get("api_key")) + return await _resolved_off_the_event_loop( + partial(_credential_updates, kwargs, credential_values, call_type=call_type), + _api_key_after_credentials(kwargs, credential_values), + ) + + +def _api_key_after_credentials(kwargs: Mapping[str, object], credential_values: Mapping[str, object]) -> object: + return kwargs.get("api_key", credential_values.get("api_key")) + + +def _credential_updates( + kwargs: Mapping[str, object], + credential_values: Mapping[str, object], + resolve_oidc: OidcResolver = resolve_api_key_oidc_reference, + call_type: str | None = None, +) -> Mapping[str, object]: + api_key: Final = _api_key_after_credentials(kwargs, credential_values) if not isinstance(api_key, AdminApiKeySecretReference): return credential_values secret: Final = _secret_behind_api_key_reference( api_key, + resolve_oidc=resolve_oidc, + call_type=call_type, model=kwargs.get("model"), llm_provider=kwargs.get("custom_llm_provider"), upstream=MappingProxyType( @@ -1539,15 +1632,19 @@ def _schedule_async_success_logging( return def _enqueue_async_logging() -> None: + from litellm.responses.streaming_iterator import spawn_logging_task + logging_obj.mark_async_success_log_enqueued() - asyncio.create_task( + spawn_logging_task( + asyncio.get_running_loop(), _client_async_logging_helper( logging_obj=logging_obj, result=result, start_time=start_time, end_time=end_time, is_completion_with_fallbacks=is_completion_with_fallbacks, - ) + ), + task_name="success callback enqueue", ) if not logging_obj.defer_async_logging: @@ -1935,7 +2032,7 @@ def client(original_function): assert logging_obj is not None, "logging_obj should not be None after function_setup" ## LOAD CREDENTIALS - load_credentials_from_list(kwargs) + load_credentials_from_list(kwargs, call_type) from litellm.llms.github_copilot.per_user_auth import attach_github_copilot_user_session attach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict @@ -2229,7 +2326,7 @@ def client(original_function): kwargs["litellm_logging_obj"] = logging_obj ## LOAD CREDENTIALS - load_credentials_from_list(kwargs) + kwargs.update(await acredential_updates(kwargs, call_type)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is the untyped request dict from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict @@ -2683,9 +2780,99 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: return None +@dataclass(frozen=True, slots=True) +class _HuggingFaceTokenizerLoaded: + tokenizer: HuggingFace + + @dataclass(frozen=True, slots=True) class _HuggingFaceTokenizerLoadFailure: reason: str + retry_at: float + retry_pid: int | None = None + + +_HuggingFaceTokenizerKey: TypeAlias = tuple[HuggingFaceTokenizerKind, Decision] +_HuggingFaceTokenizerLoadOutcome: TypeAlias = _HuggingFaceTokenizerLoaded | _HuggingFaceTokenizerLoadFailure + + +def _start_in_thread_joined_at_exit(work: Callable[[], None]) -> None: + threading.Thread(target=work, name="litellm-huggingface-tokenizer-retry", daemon=False).start() + + +@dataclass(slots=True) +class _HuggingFaceTokenizerLoadOutcomes: + clock: Callable[[], float] = time.monotonic + start_retry: Callable[[Callable[[], None]], None] = _start_in_thread_joined_at_exit + pid: Callable[[], int] = os.getpid + lock: threading.Lock = field(default_factory=threading.Lock) + by_tokenizer: Mapping[_HuggingFaceTokenizerKey, _HuggingFaceTokenizerLoadOutcome] = field( + default_factory=lambda: MappingProxyType({}) + ) + + def known( + self, key: _HuggingFaceTokenizerKey, retry_when_due: Callable[[], None] + ) -> _HuggingFaceTokenizerLoadOutcome | None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if ( + not isinstance(known, _HuggingFaceTokenizerLoadFailure) + or known.retry_pid == self.pid() + or self.clock() < known.retry_at + ): + return known + retrying: Final = replace(known, retry_pid=self.pid()) + self._remember(key, retrying) + try: + self.start_retry(retry_when_due) + except RuntimeError as error: + return self.remember_failed_retry(key, str(error)) + return retrying + + def remember_loaded(self, key: _HuggingFaceTokenizerKey, tokenizer: HuggingFace) -> _HuggingFaceTokenizerLoaded: + loaded: Final = _HuggingFaceTokenizerLoaded(tokenizer) + with self.lock: + self._remember(key, loaded) + return loaded + + def remember_first_failure(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if known is not None: + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def remember_failed_retry(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoaded): + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def end_retry(self, key: _HuggingFaceTokenizerKey) -> None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoadFailure) and known.retry_pid is not None: + self._remember(key, self._failure(known.reason)) + + def flush_cache(self) -> None: + with self.lock: + self.by_tokenizer = MappingProxyType({}) + + def _failure(self, reason: str) -> _HuggingFaceTokenizerLoadFailure: + return _HuggingFaceTokenizerLoadFailure( + reason=reason, retry_at=self.clock() + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS + ) + + def _remember(self, key: _HuggingFaceTokenizerKey, outcome: _HuggingFaceTokenizerLoadOutcome) -> None: + self.by_tokenizer = MappingProxyType({**self.by_tokenizer, key: outcome}) + + +_HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES: Final = _HuggingFaceTokenizerLoadOutcomes() def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: @@ -2697,30 +2884,28 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: ) -def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: +def _return_huggingface_tokenizer( + model: str, outcomes: _HuggingFaceTokenizerLoadOutcomes = _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES +) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): return None - backend: Final = _huggingface_tokenizer_backend() - key: Final = _huggingface_tokenizer_failure_key(kind, backend) + key: Final = (kind, _huggingface_tokenizer_backend()) with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): - cached_failure: Final = _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.get_cache(key) - if isinstance(cached_failure, _HuggingFaceTokenizerLoadFailure): - return cached_failure - tokenizer: Final = _load_huggingface_tokenizer_recording_failure(kind, backend, key) - if isinstance(tokenizer, _HuggingFaceTokenizerLoadFailure): - _warn_huggingface_tokenizer_fallback(model, tokenizer.reason) - return tokenizer - return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + known: Final = outcomes.known(key, partial(_retry_huggingface_tokenizer_load, model, key, outcomes)) + outcome: Final = ( + known + if known is not None + else _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_first_failure) + ) + match outcome: + case _HuggingFaceTokenizerLoaded(tokenizer=tokenizer): + return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + case _HuggingFaceTokenizerLoadFailure(): + return outcome -def _huggingface_tokenizer_failure_key(kind: HuggingFaceTokenizerKind, backend: Decision) -> str: - return f"{kind}:{backend.name}" - - -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: - """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> HuggingFace: match kind: case "cohere": return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") @@ -2740,19 +2925,34 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio assert_never(kind) -def _load_huggingface_tokenizer_recording_failure( - kind: HuggingFaceTokenizerKind, backend: Decision, key: str -) -> HuggingFace | _HuggingFaceTokenizerLoadFailure: +def _load_huggingface_tokenizer_remembering_outcome( + model: str, + key: _HuggingFaceTokenizerKey, + outcomes: _HuggingFaceTokenizerLoadOutcomes, + remember_failure: Callable[[_HuggingFaceTokenizerKey, str], _HuggingFaceTokenizerLoadOutcome], +) -> _HuggingFaceTokenizerLoadOutcome: + kind, _ = key try: - return _load_huggingface_tokenizer(kind, backend) + tokenizer: Final = _load_huggingface_tokenizer(kind) except Exception as error: from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): raise - failure: Final = _HuggingFaceTokenizerLoadFailure(reason=str(error)) - _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.set_cache(key, failure, ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS) - return failure + outcome: Final = remember_failure(key, str(error)) + if isinstance(outcome, _HuggingFaceTokenizerLoadFailure): + _warn_huggingface_tokenizer_fallback(model, outcome.reason) + return outcome + return outcomes.remember_loaded(key, tokenizer) + + +def _retry_huggingface_tokenizer_load( + model: str, key: _HuggingFaceTokenizerKey, outcomes: _HuggingFaceTokenizerLoadOutcomes +) -> None: + try: + _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_failed_retry) + finally: + outcomes.end_retry(key) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -3398,6 +3598,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. @@ -3712,7 +3948,7 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMa return None -def is_generalized_model_info(model_info: ModelInfo) -> bool: +def is_generalized_model_info(model_info: ModelInfoBase) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. Detected as the resolved key missing ``litellm.model_cost`` while matching a @@ -3728,10 +3964,12 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 +_ModelInfoT: Final = TypeVar("_ModelInfoT", bound=ModelInfoBase) -def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + +def _lookup_mapped_model_info(lookup: Callable[[], _ModelInfoT]) -> _ModelInfoT | ModelInfoLookupMiss: try: - info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + info: Final = lookup() except ModelNotMappedError: return ModelInfoLookupMiss(tripped=False) except Exception: @@ -3739,6 +3977,16 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + return _lookup_mapped_model_info(lambda: get_model_info(model=model, custom_llm_provider=custom_llm_provider)) + + +def lookup_catalog_model_info(model: str, custom_llm_provider: str | None) -> ModelInfoBase | ModelInfoLookupMiss: + return _lookup_mapped_model_info( + lambda: get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True) + ) + + def _get_builtin_model_info_for_registration( model: str, custom_llm_provider: str | None ) -> ModelInfo | ModelInfoLookupMiss: @@ -5004,6 +5252,46 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5093,19 +5381,30 @@ def get_optional_params( verbose_logger.debug( "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) ) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): + verbose_logger.warning( + _dropped_reasoning_warning(base_model or model, custom_llm_provider), + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + ) else: raise UnsupportedParamsError( status_code=500, @@ -6621,6 +6920,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6654,7 +6955,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6678,7 +6979,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, @@ -8346,26 +8647,6 @@ def trim_messages( from litellm.caching.in_memory_cache import InMemoryCache -class _HuggingFaceTokenizerLoadFailureCache(Protocol): - def get_cache(self, key: str) -> object | None: ... - - def set_cache(self, key: str, value: object, ttl: float | None = None) -> None: ... - - def flush_cache(self) -> None: ... - - -_HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCache] = ( - cast( # cast-ok: cache methods are untyped - _HuggingFaceTokenizerLoadFailureCache, - InMemoryCache( - max_size_in_memory=16, - default_ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, - clock=lambda: time.monotonic(), - ), - ) -) - - class AvailableModelsCache(InMemoryCache): def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): super().__init__(ttl_seconds, max_size) @@ -9771,6 +10052,12 @@ class ProviderConfigManager: ) return TencentAnthropicMessagesConfig() + elif litellm.LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.messages.transformation import ( + DashScopeAnthropicMessagesConfig, + ) + + return DashScopeAnthropicMessagesConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: if "claude" in model_lower: from litellm.llms.github_copilot.messages.transformation import ( @@ -10275,6 +10562,22 @@ class ProviderConfigManager: return BedrockVectorStore.get_initialized_custom_logger() return None + @staticmethod + def _get_vertex_ai_vector_stores_config(api_type: str | None) -> BaseVectorStoreConfig | None: + if api_type == "rag_api" or api_type is None: + from litellm.llms.vertex_ai.vector_stores.rag_api.transformation import ( + VertexVectorStoreConfig, + ) + + return VertexVectorStoreConfig() + if api_type == "search_api": + from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( + VertexSearchAPIVectorStoreConfig, + ) + + return VertexSearchAPIVectorStoreConfig() + return None + @staticmethod def get_provider_vector_stores_config( provider: LlmProviders, @@ -10296,18 +10599,7 @@ class ProviderConfigManager: return AzureOpenAIVectorStoreConfig() elif litellm.LlmProviders.VERTEX_AI == provider: - if api_type == "rag_api" or api_type is None: # default to rag_api - from litellm.llms.vertex_ai.vector_stores.rag_api.transformation import ( - VertexVectorStoreConfig, - ) - - return VertexVectorStoreConfig() - elif api_type == "search_api": - from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( - VertexSearchAPIVectorStoreConfig, - ) - - return VertexSearchAPIVectorStoreConfig() + return ProviderConfigManager._get_vertex_ai_vector_stores_config(api_type) elif litellm.LlmProviders.BEDROCK == provider: from litellm.llms.bedrock.vector_stores.transformation import ( BedrockVectorStoreConfig, @@ -10362,6 +10654,12 @@ class ProviderConfigManager: ) return MongoDBVectorStoreConfig() + elif litellm.LlmProviders.OPENSEARCH == provider: + from litellm.llms.opensearch.vector_stores.transformation import ( + OpenSearchVectorStoreConfig, + ) + + return OpenSearchVectorStoreConfig() return None @staticmethod @@ -10574,6 +10872,7 @@ class ProviderConfigManager: model: str, provider: LlmProviders, drop_params: bool = False, + litellm_params: Mapping[str, object] = MappingProxyType({}), ) -> BaseRealtimeConfig | None: if LlmProviders.GEMINI == provider: from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -10583,6 +10882,10 @@ class ProviderConfigManager: from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig return MetaRealtimeConfig(drop_params=drop_params) + if LlmProviders.SONIOX == provider: + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig + + return SonioxRealtimeConfig.from_litellm_params(litellm_params, drop_params=drop_params) return None @staticmethod diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 2b1fd029143..3ffa9140543 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -43,6 +43,7 @@ ENV UV_PYTHON_DOWNLOADS=0 COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust +COPY model_prices_and_context_window.json ./ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f36112bc0ef..dbe0be24fdc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4049,6 +4049,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true }, @@ -12437,7 +12438,8 @@ "supported_output_modalities": [ "text" ], - "supports_native_streaming": true + "supports_native_streaming": true, + "supports_pdf_input": false }, "azure_ai/FLUX-1.1-pro": { "deprecation_date": "2027-01-05", @@ -12502,6 +12504,7 @@ "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12516,6 +12519,7 @@ "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12531,6 +12535,7 @@ "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12546,6 +12551,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12560,6 +12566,7 @@ "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12574,6 +12581,7 @@ "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12594,6 +12602,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12616,6 +12625,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12638,6 +12648,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12660,6 +12671,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12687,6 +12699,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12703,6 +12716,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12724,6 +12738,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12744,6 +12759,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -12765,6 +12781,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12833,6 +12850,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -12847,6 +12865,7 @@ "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { @@ -12859,6 +12878,7 @@ "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12872,6 +12892,7 @@ "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -12884,6 +12905,7 @@ "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -12895,6 +12917,7 @@ "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -12906,6 +12929,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12918,6 +12942,7 @@ "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12930,6 +12955,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12942,6 +12968,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12954,6 +12981,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12966,6 +12994,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12978,6 +13007,7 @@ "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -12990,6 +13020,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13002,6 +13033,7 @@ "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13015,6 +13047,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": false }, @@ -13027,7 +13060,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-multimodal-instruct": { "deprecation_date": "2027-01-17", @@ -13041,6 +13075,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_audio_input": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { @@ -13052,7 +13087,8 @@ "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/Phi-4-reasoning": { "deprecation_date": "2027-01-17", @@ -13064,6 +13100,7 @@ "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_reasoning": true }, @@ -13106,6 +13143,7 @@ "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/doc-intelligence/prebuilt-read": { @@ -13144,6 +13182,7 @@ "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -13202,6 +13241,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13217,6 +13257,7 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13230,6 +13271,7 @@ "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { @@ -13242,6 +13284,7 @@ "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 1.45e-07, @@ -13257,6 +13300,7 @@ "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.8e-08, @@ -13273,6 +13317,7 @@ "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -13306,6 +13351,7 @@ "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13323,6 +13369,7 @@ "output_cost_per_token_above_200k_tokens": 5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13343,6 +13390,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -13361,6 +13409,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13379,6 +13428,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13395,6 +13445,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13409,6 +13460,7 @@ "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -13424,6 +13476,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, + "supports_pdf_input": false, "supports_response_schema": true, "supports_tool_choice": true, "supports_web_search": true @@ -13436,7 +13489,8 @@ "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false }, "azure_ai/jamba-instruct": { "deprecation_date": "2025-03-01", @@ -13447,6 +13501,7 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { @@ -13459,6 +13514,7 @@ "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true, @@ -13482,6 +13538,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, @@ -13498,6 +13555,7 @@ "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large": { @@ -13510,6 +13568,7 @@ "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-2407": { @@ -13522,6 +13581,7 @@ "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-latest": { @@ -13533,6 +13593,7 @@ "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-large-3": { @@ -13544,6 +13605,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -13557,6 +13619,7 @@ "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-nemo": { @@ -13568,7 +13631,8 @@ "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true + "supports_function_calling": true, + "supports_pdf_input": false }, "azure_ai/mistral-small": { "deprecation_date": "2025-07-31", @@ -13580,6 +13644,7 @@ "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "azure_ai/mistral-small-2503": { @@ -13591,6 +13656,7 @@ "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_vision": true }, @@ -52006,6 +52072,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52026,6 +52093,7 @@ ], "supports_assistant_prefill": false, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52777,6 +52845,7 @@ "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true, "supports_web_search": true }, @@ -52794,6 +52863,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -52811,6 +52881,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -52989,6 +53060,7 @@ "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true }, "vertex_ai/openai/gpt-oss-20b-maas": { @@ -53000,6 +53072,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_pdf_input": false, "supports_reasoning": true, "cache_read_input_token_cost": 7e-09, "input_cost_per_token_batches": 3.5e-08, @@ -53131,6 +53204,7 @@ "us-south1" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { @@ -53149,6 +53223,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": { @@ -53164,6 +53239,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": { @@ -53179,6 +53255,7 @@ "global" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_tool_choice": true }, "vertex_ai/veo-3.1-generate-preview": { @@ -61192,6 +61269,126 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview-v2": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v4": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", @@ -63670,6 +63867,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63816,6 +64014,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -67215,6 +67414,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67229,6 +67429,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -67271,6 +67472,7 @@ "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67285,6 +67487,7 @@ "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67299,6 +67502,7 @@ "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67313,6 +67517,7 @@ "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -67329,6 +67534,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -67344,6 +67550,7 @@ "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, + "supports_pdf_input": false, "supports_reasoning": true, "supports_tool_choice": true }, @@ -67362,6 +67569,7 @@ "supported_output_modalities": [ "text" ], + "supports_pdf_input": false, "supports_response_schema": true, "supports_vision": true }, @@ -67371,7 +67579,8 @@ "output_cost_per_token": 5.4e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3-0324": { "deprecation_date": "2026-07-13", @@ -67379,7 +67588,8 @@ "output_cost_per_token": 4.56e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/deepseek-v3.1": { "deprecation_date": "2026-07-13", @@ -67387,7 +67597,8 @@ "output_cost_per_token": 4.94e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3": { "deprecation_date": "2026-05-01", @@ -67395,7 +67606,8 @@ "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-3-mini": { "deprecation_date": "2026-05-01", @@ -67403,7 +67615,8 @@ "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", @@ -67411,7 +67624,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "azure_ai/grok-4-fast-reasoning": { "deprecation_date": "2026-05-01", @@ -67419,7 +67633,8 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_pdf_input": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -73089,6 +73304,7 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": ["global"], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -73704,6 +73920,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true @@ -74758,6 +74975,7 @@ "text" ], "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, @@ -75006,6 +75224,7 @@ "output_cost_per_token_above_200k_tokens": 1.2e-05, "source": "https://docs.x.ai/developers/models/grok-4.7", "supports_function_calling": true, + "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -78354,6 +78573,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 27681d9116c..db50589a675 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -56,6 +56,7 @@ editable-profile = "dev" include = [ { path = "rust-toolchain.toml", format = "sdist" }, { path = ".cargo/config.toml", format = "sdist" }, + { path = "model_prices_and_context_window.json", format = "sdist" }, "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/model_insights_tasks.json", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7b0234e28d7..1d487f33835 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3724,7 +3724,7 @@ "ocr": false, "search": false, "moderations": false, - "realtime": false, + "realtime": true, "video_generations": false, "batches": false, "messages": false, @@ -4910,6 +4910,33 @@ "vector_stores_search": true } }, + "opensearch": { + "display_name": "Amazon OpenSearch (`opensearch`)", + "url": "https://docs.litellm.ai/docs/providers/opensearch_vector_stores", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_search": true + } + }, "helicone": { "display_name": "Helicone (`helicone`)", "url": "https://docs.litellm.ai/docs/providers/helicone", @@ -5327,6 +5354,32 @@ "generateContent": false } }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "bigmodel": { "display_name": "BigModel (Zhipu AI) (`bigmodel`)", "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", diff --git a/pyproject.toml b/pyproject.toml index 200bb463313..e6d34a3d90c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -335,6 +335,7 @@ include = [ "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", + { path = "model_prices_and_context_window.json", format = "sdist" }, ] exclude = [ "litellm/proxy/enterprise", @@ -357,6 +358,7 @@ cache-keys = [ { file = "litellm-rust/Cargo.lock" }, { file = "litellm-rust/Cargo.toml" }, { file = "litellm-rust/crates/**/*" }, + { file = "model_prices_and_context_window.json" }, ] constraint-dependencies = [ "tornado>=6.5.8", diff --git a/scripts/build_core_distribution.py b/scripts/build_core_distribution.py index 36ec20a9031..41d7208d2c3 100644 --- a/scripts/build_core_distribution.py +++ b/scripts/build_core_distribution.py @@ -9,7 +9,15 @@ from pathlib import Path from typing import Final ROOT: Final = Path(__file__).resolve().parents[1] -SOURCES: Final = ("litellm", "litellm-rust", ".cargo", "rust-toolchain.toml", "README.md", "LICENSE") +SOURCES: Final = ( + "litellm", + "litellm-rust", + ".cargo", + "rust-toolchain.toml", + "model_prices_and_context_window.json", + "README.md", + "LICENSE", +) def stage_core_distribution(source: Path, destination: Path) -> None: diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 371884fb791..95b5942fa8b 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -18,7 +18,8 @@ # -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's # test-tree ruff and test-quality gate steps) # - dashboard -> prettier + eslint + lint budgets (test-linting.yml's ui job) -# - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (test-linting.yml's ui-api-types job) +# - spec_pattern -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift +# (test-linting.yml's ui-api-types job) # # Each block is skipped when no matching files are in scope, so unrelated commits # stay fast. This is intentionally not auto-installed as a git hook (see @@ -98,7 +99,7 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e(_harness)?/.*\.py$' test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|scripts/(check_test_quality|test_quality_gate)\.py)$' -spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' +spec_pattern='^(litellm/.*|enterprise/.*|pyproject\.toml|uv\.lock|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' @@ -110,9 +111,6 @@ e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files) -# test-linting.yml triggers on any file under litellm/proxy or litellm/types -# (Prisma schema and configs included, not just Python) plus the generator and its -# lockfiles, so match that whole trigger set rather than a Python subset. spec_files=$(scope_match "$spec_pattern") # CI's ui job runs prettier over a wider extension set than eslint; keep that # split so this flags exactly what the job would. @@ -337,7 +335,7 @@ summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_fi summary_item "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_files" \ "no tests/ Python files or test-tree lint inputs in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" -summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" +summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/, enterprise/, dependency, or generator files in scope" if [ -z "$litellm_py_files$e2e_py_files$test_tree_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2 diff --git a/tests/base_sdk_tests/test_core_distribution.py b/tests/base_sdk_tests/test_core_distribution.py index ba68d08e4d1..c22fa4e54fc 100644 --- a/tests/base_sdk_tests/test_core_distribution.py +++ b/tests/base_sdk_tests/test_core_distribution.py @@ -132,7 +132,9 @@ def test_core_artifacts_exclude_local_configuration(distributions: tuple[Path, P assert not any(member.name.endswith(f"/{name}") for member in archive.getmembers()) -@pytest.mark.parametrize("relative_path", ["rust-toolchain.toml", ".cargo/config.toml"]) +@pytest.mark.parametrize( + "relative_path", ["rust-toolchain.toml", ".cargo/config.toml", "model_prices_and_context_window.json"] +) def test_core_sdist_preserves_native_build_configuration(distribution_directory: Path, relative_path: str) -> None: sdists: Final = tuple(distribution_directory.glob("litellm_core-*.tar.gz")) assert len(sdists) == 1 diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 330ddf88fc5..367135eae26 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -2,6 +2,7 @@ - {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"} - {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"} - {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"} +- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings, refuse chat with a gateway 400, and route chat through a live fallback; zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"} - {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"} - {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"} - {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"} diff --git a/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py b/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py new file mode 100644 index 00000000000..c45adfeeb84 --- /dev/null +++ b/tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py @@ -0,0 +1,119 @@ +"""Live provider checks for embeddings, the chat refusal, and chat fallback from an explicitly restricted model.""" + +from __future__ import annotations + +import dataclasses +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody +from openai import BadRequestError, OpenAI +from proxy_client import ProxyClient +from sdk_clients import NO_PROXY_CACHE, SdkClients + +pytestmark = pytest.mark.e2e + +EMBEDDING_BACKEND: Final = "openai/text-embedding-3-small" +CHAT_FALLBACK_BACKEND: Final = "openai/gpt-5.4-nano" +EMBEDDINGS_ONLY: Final = ("/v1/embeddings",) +PROMPT: Final = "Say this is a test!" + + +def _subject(route: Route) -> Subject: + return Subject( + domain=Domain.LLM_TRANSLATION, + route=route, + providers=(Provider.OPENAI,), + models=(EMBEDDING_BACKEND,), + mode=Mode.NONSTREAM, + ) + + +def _register( + proxy: ProxyClient, + resources: ResourceManager, + prefix: str, + supported_endpoints: tuple[str, ...] | None, + backend: str = EMBEDDING_BACKEND, + mode: ModelMode = "embedding", +) -> str: + model: Final = f"{prefix}-{unique_marker()}" + model_id: Final = proxy.register_model( + ModelNewBody( + model_name=model, + litellm_params=LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY"), + model_info=ModelInfoBody( + enforce_supported_endpoints=supported_endpoints is not None, + mode=mode, + supported_endpoints=None if supported_endpoints is None else list(supported_endpoints), + ), + ) + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return model + + +def _openai_at(sdk: SdkClients, api_prefix: str, key: str) -> OpenAI: + return dataclasses.replace(sdk, base_url=f"{sdk.base_url}{api_prefix}").openai(key) + + +class TestEmbeddingOnlySupportedEndpoints: + @pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works") + @meta(_subject(Route.EMBEDDINGS)) + @pytest.mark.parametrize( + ("prefix", "supported_endpoints"), + ( + pytest.param("e2e-embed-only", EMBEDDINGS_ONLY, id="embed-only"), + pytest.param("e2e-embed-control", None, id="control"), + ), + ) + def test_embeddings_return_a_vector( + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + prefix: str, + supported_endpoints: tuple[str, ...] | None, + ) -> None: + model: Final = _register(proxy, resources, prefix, supported_endpoints) + embeddings: Final = sdk.openai(resources.key()).embeddings.create( + model=model, input=PROMPT, extra_body=NO_PROXY_CACHE + ) + assert embeddings.data, f"/embeddings returned no data for {model}: {embeddings!r}" + assert any(component != 0.0 for component in embeddings.data[0].embedding), ( + f"/embeddings returned an empty or all-zero vector for {model}" + ) + + @pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works") + @meta(dataclasses.replace(_subject(Route.CHAT_COMPLETIONS), models=(EMBEDDING_BACKEND, CHAT_FALLBACK_BACKEND))) + def test_chat_completions_with_a_fallback_are_answered_by_the_fallback( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY) + fallback: Final = _register(proxy, resources, "e2e-chat-fallback", None, CHAT_FALLBACK_BACKEND, "chat") + completion: Final = _openai_at(sdk, "/v1", resources.key()).chat.completions.create( + model=model, + messages=[{"role": "user", "content": PROMPT}], + extra_body={**NO_PROXY_CACHE, "fallbacks": [fallback]}, + ) + assert completion.choices and completion.choices[0].message.content, ( + f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}" + ) + + @pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works") + @meta(_subject(Route.CHAT_COMPLETIONS)) + def test_chat_completions_without_a_fallback_are_refused_by_the_gateway( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY) + with pytest.raises(BadRequestError) as refused: + _ = _openai_at(sdk, "/v1", resources.key()).chat.completions.create( + model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE + ) + assert ( + f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings" + in refused.value.message + ), f"chat on {model} should be refused by the gateway before any provider call, got {refused.value!r}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index afddc7e25d6..504adec5401 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1416,7 +1416,7 @@ class LiteLLMParamsBody(BaseModel): order: int | None = None -ModelMode = Literal["chat", "batch", "realtime", "image_generation"] +ModelMode = Literal["chat", "embedding", "batch", "realtime", "image_generation"] class TokenCountModelBody(BaseModel, frozen=True): @@ -1425,6 +1425,7 @@ class TokenCountModelBody(BaseModel, frozen=True): class ModelInfoBody(BaseModel): + enforce_supported_endpoints: bool | None = None # id is left unset so the proxy assigns a unique model_id per deployment. # Pinning it to the model_name made re-registrations of a fixed-name model # (e.g. the batch suite's openai-batch) collide on the model_id unique diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index a727b20596c..5bdc538b2d9 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -37,3 +37,14 @@ together with its dedicated job, and list a new dedicated GitHub workflow with t `DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its directory (#46780) + +## Fixtures that boot a proxy + +Scope them to the module or class, or pin a session one to a single worker with an `xdist_group` mark +as `_llm_gateway` in `mcp/test_mcp_llm_endpoints.py` does, and request them as arguments, never through +`request.getfixturevalue`. The harness puts tests that request the same set of module or class fixtures +on one xdist worker, so the proxy they share boots there once and stops when that worker moves to another +file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot +the proxy again on another worker, so keep that set the same across the tests that share a proxy. An +unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until +the node ends, and those copies ran CircleCI integration-extensions nodes out of memory diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index b8ea11f3d2f..a759126fcd4 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -280,12 +280,14 @@ class Scenario: self.gateway.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == [] ) - def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str: - name: Final = f"integration-{uuid.uuid4().hex}" + def model( + self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue + ) -> str: + model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}" created: Final = self.gateway.post( "/model/new", { - "model_name": name, + "model_name": model_name, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", @@ -298,8 +300,8 @@ class Scenario: identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) if self.gateway.workers > 1: - self._await_every_worker(name, identity) - return name + self._await_every_worker(model_name, identity) + return model_name def _await_every_worker(self, name: str, identity: str) -> None: from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 9b990fd2167..55519678f40 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -73,6 +73,7 @@ COPILOT_USER_CODE: Final = "COPI-LEAK" COPILOT_ACCESS: Final = "device-login-github-access" COPILOT_REJECTED_ACCESS: Final = "device-login-github-access-rejected" COPILOT_FIRST_LOGIN_ACCESS: Final = "device-login-github-access-first-login" +COPILOT_UNSAVED_KEY_ACCESS: Final = "device-login-github-access-unsaved-key" COPILOT_STORED_KEY: Final = "tid=device-login;stored" COPILOT_MINTED_KEY: Final = "tid=device-login;minted" COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected" @@ -132,7 +133,11 @@ def _oauth_token(request: Request, grant: threading.Event) -> Reply: def _copilot_api_key(request: Request, api_url: str) -> Reply: - if request.headers.get("authorization") not in (f"token {COPILOT_ACCESS}", f"token {COPILOT_FIRST_LOGIN_ACCESS}"): + if request.headers.get("authorization") not in ( + f"token {COPILOT_ACCESS}", + f"token {COPILOT_FIRST_LOGIN_ACCESS}", + f"token {COPILOT_UNSAVED_KEY_ACCESS}", + ): return _json(401, {"message": "Bad credentials"}) return _json( 200, @@ -380,4 +385,7 @@ def write_copilot_access(directory: Path, access_token: str) -> None: def clear_tokens(*directories: Path) -> None: for directory in directories: for path in directory.iterdir(): - path.unlink() + if path.is_dir(): + path.rmdir() + else: + path.unlink() diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index a80eeacad46..4dedbdcd8e4 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -545,3 +545,94 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be assert set(token_form) == {"code", "code_verifier", "grant_type", "redirect_uri"}, token_form assert userinfo_request.method == "GET" and urlsplit(userinfo_request.target).path == "/userinfo" assert user_id == _USER_ID + + +_TEAM_OPTION: Final = re.compile(r'