diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 158110cbbab..2826098fd1f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,6 @@ /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri +/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /helm/ @yassin-berriai /terraform/ @yassin-berriai /.github/workflows/zizmor.yml @yuneng-berri diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index 05b22017cbb..6519b65e71b 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -21,16 +21,18 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - - name: Fetch the pull request head and its merge base + - name: Fetch the pull request head, its merge base, and the base branch tip id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" - git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" + target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')" + merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" + git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + echo "target=$target" >> "$GITHUB_OUTPUT" - name: Set up uv uses: ./.github/actions/setup-uv-with-retries with: @@ -38,6 +40,8 @@ jobs: - name: Run the guard env: MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} + COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }} + COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | diff --git a/.github/workflows/cost-map-pins-recheck.yml b/.github/workflows/cost-map-pins-recheck.yml new file mode 100644 index 00000000000..f2572efbb31 --- /dev/null +++ b/.github/workflows/cost-map-pins-recheck.yml @@ -0,0 +1,91 @@ +name: Cost map pins recheck + +on: + push: + branches: + - main + - "litellm_**" + paths: + - model_prices_and_context_window.json + - litellm/model_prices_and_context_window_backup.json + - ci_cd/cost_map_pins.json + +permissions: {} + +jobs: + rerun-cost-map-guard: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: write + contents: read + pull-requests: read + steps: + - name: Re-run the cost map guard on open pull requests the push could break + env: + GH_TOKEN: ${{ github.token }} + BASE_REF: ${{ github.ref_name }} + BEFORE_SHA: ${{ github.event.before }} + AFTER_SHA: ${{ github.event.after }} + FORCED: ${{ github.event.forced }} + COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]' + PINS_PATHS: '["ci_cd/cost_map_pins.json"]' + run: | + if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \ + || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \ + --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \ + | grep -qx true; then + export RECHECK_PATHS="$COST_MAP_PATHS" + else + export RECHECK_PATHS="$PINS_PATHS" + fi + rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)" + candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \ + -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) { + nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } } + pageInfo { hasNextPage endCursor } + } + } + }' \ + --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[] + | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)} + | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')" + failed=0 + declare -A rerun_heads + while read -r number head_sha listed; do + [ -n "$number" ] || continue + if [ "$listed" = false ]; then + if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \ + --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then + echo "::error::could not list the files of #$number" + failed=1 + continue + fi + [ -n "$changed" ] || continue + fi + [ -z "${rerun_heads[$head_sha]:-}" ] || continue + rerun_heads[$head_sha]=1 + if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then + echo "::error::could not list the cost map guard runs of #$number" + failed=1 + continue + fi + if [ -z "$runs" ]; then + echo "::warning::no cost map guard run found for #$number" + continue + fi + while read -r run_id created_at; do + if [[ "$created_at" < "$rerun_cutoff" ]]; then + echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push" + continue + fi + gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true + if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then + echo "::error::could not re-run cost map guard run $run_id for #$number" + failed=1 + fi + done <<< "$runs" + done <<< "$candidates" + exit "$failed" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 89ca60ad9d1..bd7343ea222 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -376,10 +376,10 @@ jobs: exit 0 fi files="$(git diff --name-only "$base" HEAD)" - if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then + if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then echo "relevant=true" >> "$GITHUB_OUTPUT" else - echo "No proxy, types or generator changes in this pull request, nothing to verify." + echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify." echo "relevant=false" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index ff9db13bd25..6aeb66ed37c 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -12,6 +12,7 @@ on: paths: - "terraform/provider/**" - "litellm/proxy/**" + - "enterprise/**" - ".github/workflows/test-terraform-provider.yml" permissions: diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4768eeb633e..e7865817048 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -236,6 +236,7 @@ jobs: tests/unit/llms/azure_ai tests/unit/llms/base_llm tests/unit/llms/baseten + tests/unit/llms/bigmodel tests/unit/llms/black_forest_labs tests/unit/llms/brave tests/unit/llms/bytez diff --git a/.grype.yaml b/.grype.yaml index c5e49851dc9..dedc8fd528e 100644 --- a/.grype.yaml +++ b/.grype.yaml @@ -11,3 +11,12 @@ ignore: package: name: zlib type: apk + - vulnerability: GHSA-v4x8-3wr5-3wq8 + reason: >- + filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock; + the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it, + and the image then moves off 3.32.7, which retires this version-scoped rule + package: + name: filelock + version: 3.32.7 + type: python diff --git a/AGENTS.md b/AGENTS.md index 355e300f129..ce4b63173d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - No file sprawl: deliberate file and folder structure - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration +- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated Follow conventional commits for commit names and PR titles diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index c1aeaabc5d2..d5fd755ba18 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,37 +1,67 @@ """Guard the cost map on pull requests. -Every pull request whose diff against its merge base touches one of the two cost map files gets the file -checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched -skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the -branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always -get the file checks and additionally may only touch those two files and may only add or update models. +Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the +file checks: the files parse, the backup copy matches the root file, and every capability pinned in +ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says +otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before +a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves +all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries +whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named +litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may +only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests +(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is +Bot) get the human checks and may not touch the pins file either. """ from __future__ import annotations import argparse import json +import os import subprocess import sys -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" +PINS_PATH: Final = "ci_cd/cost_map_pins.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) +CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" +PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" +TARGET_ENV: Final = "COST_MAP_GUARD_TARGET" +AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE" +BOT_AUTHOR_TYPE: Final = "Bot" +MERGE_CONFLICT_EXIT_CODE: Final = 1 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) CostMap = dict[str, object] +_MISSING: Final = object() @dataclass(frozen=True, slots=True) class Snapshot: cost_map: str backup: str + pins: str + + +@dataclass(frozen=True, slots=True) +class MergeFailed: + stderr: str + + +@dataclass(frozen=True, slots=True) +class Pin: + key: str + field: str + value: object + checked: str + source: str def _parse_object(text: str, path: str) -> CostMap | str: @@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]: return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)} +def _pin(key: str, field: str, spec: object) -> Pin | str: + malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source" + if not isinstance(spec, dict) or "value" not in spec: + return malformed + checked: Final = spec.get("checked") + source: Final = spec.get("source") + if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source: + return malformed + return Pin(key, field, spec["value"], checked, source) + + +def _pins(parsed: CostMap) -> Iterator[Pin | str]: + for key, fields in parsed.items(): + if not isinstance(fields, dict): + yield f"{PINS_PATH}: {key} must map field names to pins" + continue + for field, spec in fields.items(): + yield _pin(key, field, spec) + + +def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]: + for pin in _pins(parsed): + if isinstance(pin, str): + yield pin + elif pin.key not in entries: + yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry" + elif entries[pin.key].get(pin.field, _MISSING) != pin.value: + yield ( + f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} " + f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first" + ) + + +def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]: + if not pins_text: + return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",) + parsed: Final = _parse_object(pins_text, PINS_PATH) + if isinstance(parsed, str): + return (parsed,) + return tuple(_pin_violations(parsed, _entries(head_map))) + + def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]: base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH) if isinstance(base_map, str): @@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) +def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]: + if PINS_PATH not in changed_files: + return () + if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX): + return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) + if author_type == BOT_AUTHOR_TYPE: + return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) + return () + + def touches_cost_map(changed_files: Sequence[str]) -> bool: - return any(path in GUARDED_PATHS for path in changed_files) + return any(path in CHECKED_PATHS for path in changed_files) def contract_for(bot: bool, changed_files: Sequence[str]) -> str: @@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str: return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" +def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]: + if merged is None: + return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",) + if not merged.pins and not base.pins: + return () + merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH) + if isinstance(merged_map, str): + return (f"after merging into the base branch, {merged_map}",) + return pin_failures(merged.pins, merged_map) + + def guard_failures( base: Snapshot, head: Snapshot, + merged: Snapshot | None, changed_files: Sequence[str], bot: bool, *, @@ -147,6 +241,7 @@ def guard_failures( *_file_failures( head, head_map, evaluation_output_price_sources=evaluation_output_price_sources ), + *_merged_pin_failures(base, merged), *(_bot_failures(base, head_map, changed_files) if bot else ()), ) @@ -157,7 +252,22 @@ def _git(*args: str) -> str | None: def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) + + +def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None: + result: Final = subprocess.run( + ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head), + check=False, + capture_output=True, + text=True, + ) + tree: Final = result.stdout.split("\n", 1)[0] + if result.returncode == 0: + return tree + if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree: + return None + return MergeFailed(result.stderr.strip()) def changed_files(base: str, head: str) -> tuple[str, ...] | None: @@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", required=True, help="merge base of the pull request") parser.add_argument("--head", required=True, help="head commit of the pull request") + env_target: Final = os.environ.get(TARGET_ENV) + parser.add_argument( + "--target", + default=env_target, + required=env_target is None, + help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}", + ) parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) @@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int: if changed is None: print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") return 1 - failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot) + if not bot and not touches_cost_map(changed): + print(f"cost map guard passed ({contract_for(bot, changed)})") + return 0 + merged: Final = merged_tree(args.base, args.target, args.head) + if isinstance(merged, MergeFailed): + print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}") + return 1 + failures: Final = ( + *guard_failures( + snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot + ), + *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed), + ) contract: Final = contract_for(bot, changed) if failures: print(f"cost map guard failed ({contract}):") diff --git a/ci_cd/cost_map_pins.json b/ci_cd/cost_map_pins.json new file mode 100644 index 00000000000..780cd5f1276 --- /dev/null +++ b/ci_cd/cost_map_pins.json @@ -0,0 +1,16 @@ +{ + "fireworks_ai/accounts/fireworks/models/minimax-m3": { + "supports_vision": { + "value": true, + "checked": "2026-09-26", + "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row" + } + }, + "fireworks_ai/minimax-m3": { + "supports_vision": { + "value": true, + "checked": "2026-09-26", + "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row" + } + } +} diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 07048cf2c2f..7baef40e86b 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Final, cast +from typing import TYPE_CHECKING, Final from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( ManagedObjectCreatorAttribution, @@ -290,12 +290,14 @@ class CheckResponsesCost: sees provider env vars, so it fails for every deployment whose credentials live in the config; the row then never leaves ``queued``. """ - if not via_router: - return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) - router_response = await self.llm_router.aget_responses( - response_id=response_id, litellm_metadata=litellm_metadata + retrieved: Final[object] = ( + await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) + if via_router + else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) ) - return cast(ResponsesAPIResponse, router_response) + if not isinstance(retrieved, ResponsesAPIResponse): + raise TypeError(f"Retrieving {response_id} did not return a response object") + return retrieved async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse: return await self._get_response( diff --git a/examples/bigmodel.md b/examples/bigmodel.md new file mode 100644 index 00000000000..88a69796ebe --- /dev/null +++ b/examples/bigmodel.md @@ -0,0 +1,63 @@ +# BigModel (Zhipu AI) + +Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL + +```yaml +model_list: + - model_name: glm + litellm_params: + model: bigmodel/glm-5.3 + api_key: os.environ/BIGMODEL_API_KEY + api_base: https://open.bigmodel.cn +``` + +One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API + +| Proxy endpoint | BigModel endpoint | +| --- | --- | +| `/v1/chat/completions` | `/api/paas/v4/chat/completions` | +| `/v1/responses` | `/api/v1/responses` | +| `/v1/messages` | `/api/anthropic/v1/messages` | + +`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix + +A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above + +The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` + +```python +import litellm + +chat = litellm.completion( + model="bigmodel/glm-5.3", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "enabled"}, + reasoning_effort="low", +) + +response = litellm.responses( + model="bigmodel/glm-5.3", + input="Hello", + reasoning={"effort": "low"}, + store=True, +) + +async def messages(): + return await litellm.anthropic.messages.acreate( + model="bigmodel/glm-5.3", + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "adaptive"}, + output_config={"effort": "low"}, + ) +``` + +Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route + +GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters + +Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search + +The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices + +API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/litellm/__init__.py b/litellm/__init__.py index fe11884a484..a1bda4aa9f7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -39,6 +39,7 @@ from litellm._logging import ( turn_on_json, log_level, ) +from litellm._logging import suppress_startup_output as suppress_startup_output import re from litellm.constants import ( DEFAULT_BATCH_SIZE, @@ -591,7 +592,7 @@ vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set -open_ai_chat_completion_models: Set = set() +open_ai_chat_completion_models: Set[str] = set() open_ai_text_completion_models: Set = set() cohere_models: Set = set() cohere_chat_models: Set = set() @@ -619,6 +620,8 @@ vertex_openai_models: Set = set() vertex_minimax_models: Set = set() vertex_moonshot_models: Set = set() vertex_zai_models: Set = set() +vertex_qwen_models: Final[Set[str]] = set() +vertex_ai_models: Final[Set[str]] = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -635,6 +638,7 @@ watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() +bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime deepseek_models: Set = set() tencent_models: Set = set() runwayml_models: Set = set() @@ -681,6 +685,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() @@ -833,6 +838,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje elif value.get("litellm_provider") == "vertex_ai-zai_models": key = key.replace("vertex_ai/", "") vertex_zai_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-qwen_models": + vertex_qwen_models.add(key.removeprefix("vertex_ai/")) + elif value.get("litellm_provider") == "vertex_ai": + vertex_ai_models.add(key.removeprefix("vertex_ai/")) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) @@ -877,6 +886,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje xai_models.add(key) elif value.get("litellm_provider") == "zai": zai_models.add(key) + elif value.get("litellm_provider") == "bigmodel": + bigmodel_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -971,6 +982,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "kyma": + kyma_models.add(key) elif value.get("litellm_provider") == "streamlake": streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": @@ -1107,6 +1120,7 @@ model_list = list( | text_completion_inception_models | xai_models | zai_models + | bigmodel_models | fal_ai_models | deepseek_models | modelscope_models @@ -1148,6 +1162,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | kyma_models | streamlake_models | zerogpu_models | v0_models @@ -1204,7 +1219,16 @@ def _build_models_by_provider() -> dict: | vertex_deepseek_models | vertex_minimax_models | vertex_moonshot_models - | vertex_zai_models, + | vertex_zai_models + | vertex_qwen_models + | vertex_mistral_models + | vertex_llama3_models + | vertex_ai_ai21_models + | vertex_openai_models + | vertex_ai_image_models + | vertex_ai_video_models + | vertex_embedding_models + | vertex_ai_models, "ai21": ai21_models | ai21_chat_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, @@ -1221,6 +1245,7 @@ def _build_models_by_provider() -> dict: "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, + "bigmodel": bigmodel_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "tencent": tencent_models, @@ -1269,6 +1294,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "kyma": kyma_models, "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 244efc8047b..c2243b90fbf 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families" _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) +_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) + @contextmanager def internal_sub_call() -> Generator[None]: @@ -123,6 +125,30 @@ def current_service_caller() -> str | None: return _service_caller.get() +@contextmanager +def mirrored_call() -> Generator[None]: + """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included.""" + token: Final = _mirrored_call.set(True) + try: + yield + finally: + _restore(_mirrored_call, token) + + +def in_mirrored_call() -> bool: + return _mirrored_call.get() + + +@contextmanager +def outside_mirrored_call() -> Generator[None]: + """Logging a mirrored call: the calls its callbacks start are not mirrors themselves.""" + token: Final = _mirrored_call.set(False) + try: + yield + finally: + _restore(_mirrored_call, token) + + @contextmanager def pinned_billing_time(moment: datetime) -> Generator[None]: """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" diff --git a/litellm/_logging.py b/litellm/_logging.py index 8974f85f517..5279cb233b4 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -10,10 +10,12 @@ import sys from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter +from types import FrameType from typing import Final, TextIO from urllib.parse import unquote from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( @@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s" _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s" _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}" +_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") def _stream_is_tty(stream: TextIO | None) -> bool: @@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool: return False -def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: - """The plain-text log format, colorized only when both streams are an interactive terminal. +def _color_enabled(stream: TextIO | None) -> bool: + """True when ANSI colors written to the stream reach an interactive terminal. Honors the NO_COLOR convention from no-color.org: color is disabled when NO_COLOR is present with a non-empty value. """ - if os.environ.get("NO_COLOR"): - return _PLAIN_LOG_FORMAT - return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT + return not os.environ.get("NO_COLOR") and _stream_is_tty(stream) + + +def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool: + """True when both streams are an interactive color terminal, the one rule for log lines and startup prints.""" + return _color_enabled(stdout) and _color_enabled(stderr) + + +def terminal_text(message: str) -> str: + """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses.""" + return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message) + + +def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: + """The plain-text log format, colorized only when both streams are an interactive terminal.""" + return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT class LevelRoutingStreamHandler(logging.StreamHandler): @@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler): super().emit(record) -def _parse_json_logs_env(value: str | None) -> bool: - """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs. +def _parse_env_flag(value: str | None) -> bool: + """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it. Matches the reader in litellm-proxy-extras/_logging.py. The previous bool(os.getenv(...)) treated any non-empty value, including "false" and "0", @@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int: return getattr(logging, log_level.upper()) -json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS")) +json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS")) +suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT")) + + +class StartupOutputSettings(TypedDict, total=False): + json_logs: ReadOnly[object] + suppress_startup_output: ReadOnly[object] + + +STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings) + + # Create a handler for the logger (you may need to adapt this based on your needs) log_level: Final = os.getenv("LITELLM_LOG", "DEBUG") numeric_level: Final[int] = resolve_log_level(log_level) @@ -743,7 +770,7 @@ class JsonFormatter(Formatter): return dt.isoformat() def format(self, record): - message_str: Final = record.getMessage() + message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage()) json_record: Final[dict[str, object]] = { "message": message_str, "level": record.levelname, @@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter): """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter. Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs - behaves the same whether or not json_logs is enabled. + behaves the same whether or not json_logs is enabled. Unless it was built with the color format, + it also drops the ANSI color codes a message carries, which a piped log would store as text. """ def format(self, record: logging.LogRecord) -> str: rendered: Final = super().format(record) - formatted: Final = rendered if _is_redacted(record) else redact_string(rendered) + uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered) + formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored) trace_id: Final = getattr(record, "trace_id", None) session_id: Final = getattr(record, "session_id", None) if not trace_id and not session_id: @@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter) verbose_proxy_stdout_logger.addFilter(_diagnostic_filter) verbose_logger.addFilter(_diagnostic_filter) -_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") - def print_startup_message(message: str) -> None: """Write a boot-time message meant for the operator's terminal. - Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of - stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger - instead, on stdout like the print whatever handlers the logging config gave that logger, with the - ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log - level, whole and unredacted as the print showed it; a blank spacer produces no record. + LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them + off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing, + keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a + container log never stores escape codes. + With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one + INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging + config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at + the caller and emitted at any log level, whole and unredacted as the print showed it; a blank + spacer produces no record. """ + if litellm.suppress_startup_output is True: + return + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO) + + +def print_startup_warning(message: str) -> None: + """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output + on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on + stdout where the print put it, so a collector that filters on the level field keeps it.""" + frame: Final = inspect.currentframe() + _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING) + + +def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None: if not litellm.json_logs: - print(message) # noqa: T201 # plain mode: this is the operator-terminal print it replaces + import click + + click.echo(terminal_text(message)) return text: Final = _ANSI_ESCAPE.sub("", message).strip("\n") if not text: return - frame: Final = inspect.currentframe() - caller: Final = frame.f_back if frame is not None else None record: Final = verbose_proxy_logger.makeRecord( verbose_proxy_logger.name, - logging.INFO, + level, caller.f_code.co_filename if caller is not None else __file__, caller.f_lineno if caller is not None else 0, text, @@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None: def _startup_json_handler() -> logging.Handler: - """The proxy's JSON handler on stdout, filters included, for one startup message.""" - handler: Final = LevelRoutingStreamHandler() + """The proxy's JSON handler on stdout, filters included, for one startup message or warning.""" + handler: Final = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) handler.addFilter(_secret_filter) handler.addFilter(_correlation_filter) diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py new file mode 100644 index 00000000000..fc36699bc93 --- /dev/null +++ b/litellm/anthropic_beta_gated_fields.py @@ -0,0 +1,325 @@ +"""Body fields that an ``anthropic-beta`` value gates. + +``anthropic_beta_headers_config.json`` says which beta each provider leg +carries. When a leg drops a beta, the request fields that beta unlocks +would make the provider 400. Under ``drop_params`` the same request degrades on +that leg by dropping those fields too; without it the request goes out as sent +and the provider answers its own 400, like any unsupported param. Every helper +here is copy-on-write: the caller's containers are returned untouched when +nothing is stripped, and a shallow copy is returned otherwise, so a fallback +leg still sends the full request. +""" + +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) + +ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" +ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" + +THINKING_BLOCK_BINDING: Final = "thinking.block_binding" +THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" +SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" +SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" +SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" +SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" + +_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) + +BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,), + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,), + ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,), + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + } +) + + +def _betas_unlocking(field: str) -> frozenset[str]: + return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields) + + +_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields + } # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index +) + + +def field_is_gated(field: str, active_betas: Set[str]) -> bool: + """True when no active beta unlocks ``field``, so the field must be stripped.""" + return _FIELD_BETAS[field].isdisjoint(active_betas) + + +_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) +_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING_ADAPTER.validate_python(value, strict=True) + except ValidationError: + return None + + +def _as_sequence(value: object) -> tuple[object, ...] | None: + try: + return _SEQUENCE_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _is_tool_change_block(block: object) -> bool: + mapping: Final = _as_mapping(block) + block_type: Final = mapping.get("type") if mapping is not None else None + return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES + + +def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]: + joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta") + return frozenset(piece.strip() for piece in joined.split(",") if piece.strip()) + + +def _validated_headers(headers: object) -> Mapping[str, str]: + try: + return _HEADERS_ADAPTER.validate_python(headers) + except ValidationError: + return MappingProxyType({}) + + +def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]: + """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched + case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body.""" + body_values: Final = _as_sequence(body_betas) or () + body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str)) + return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings + + +@dataclass(frozen=True, slots=True) +class StrippedThinking: + thinking: object + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedMessages: + messages: Sequence[object] + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedRequest: + request: Mapping[str, object] + stripped: tuple[str, ...] + + +def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking: + """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated.""" + mapping: Final = _as_mapping(thinking) + if mapping is None or not drops_unsupported_params(drop_params): + return StrippedThinking(thinking=thinking, stripped=()) + binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas) + display_gated: Final = mapping.get("display") == "updates" and field_is_gated( + THINKING_DISPLAY_UPDATES, active_betas + ) + if not binding_gated and not display_gated: + return StrippedThinking(thinking=thinking, stripped=()) + gated_keys: Final = frozenset( + {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())} + ) + stripped: Final = ( + *((THINKING_BLOCK_BINDING,) if binding_gated else ()), + *((THINKING_DISPLAY_UPDATES,) if display_gated else ()), + ) + return StrippedThinking( + thinking={key: value for key, value in mapping.items() if key not in gated_keys}, + stripped=stripped, + ) + + +def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages: + """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing + when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400.""" + clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas) + output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas) + content: Final = message.get("content") + blocks: Final = _as_sequence(content) if not isinstance(content, str) else None + tool_changes_gated: Final = ( + blocks is not None + and any(_is_tool_change_block(block) for block in blocks) + and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas) + ) + if not clear_at_gated and not output_config_gated and not tool_changes_gated: + return StrippedMessages(messages=(message,), stripped=()) + kept_content: Final[object] = ( + [block for block in blocks if not _is_tool_change_block(block)] + if tool_changes_gated and blocks is not None + else content + ) + gated_keys: Final = frozenset( + {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())} + ) + rewritten: Final = { + key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys + } + stripped: Final = ( + *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()), + *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()), + *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()), + ) + if kept_content in (None, "", []) and "output_config" not in rewritten: + return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED)) + return StrippedMessages(messages=(rewritten,), stripped=stripped) + + +def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages: + mapping: Final = _as_mapping(message) + if mapping is None or mapping.get("role") != "system": + return StrippedMessages(messages=(message,), stripped=()) + return _strip_beta_gated_system_message(mapping, active_betas) + + +def _ends_on_assistant(messages: Sequence[object]) -> bool: + last: Final = _as_mapping(messages[-1]) if messages else None + return last is not None and last.get("role") == "assistant" + + +def strip_beta_gated_system_messages( + messages: Sequence[object], active_betas: Set[str], *, drop_params: object +) -> StrippedMessages: + """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same + sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation + ending on an assistant turn, which the model would read as a prefill to continue.""" + if not drops_unsupported_params(drop_params): + return StrippedMessages(messages=messages, stripped=()) + per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages) + stripped: Final = tuple( + name for result in per_message for name in result.stripped + ) # comprehension-ok: flattening per-message strip reports + if not stripped: + return StrippedMessages(messages=messages, stripped=()) + kept: Final = [ + message for result in per_message for message in result.messages + ] # comprehension-ok: flattening per-message results + if _ends_on_assistant(kept) and not _ends_on_assistant(messages): + return StrippedMessages(messages=messages, stripped=()) + return StrippedMessages(messages=kept, stripped=stripped) + + +def strip_beta_gated_request_fields( + request: Mapping[str, object], active_betas: Set[str], *, drop_params: object +) -> StrippedRequest: + """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) + messages: Final = request.get("messages") + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped) + if not stripped: + return StrippedRequest(request=request, stripped=()) + rewritten: Final = { + **request, + **({"thinking": thinking.thinking} if thinking.stripped else {}), + **({"messages": system_messages.messages} if system_messages.stripped else {}), + } + return StrippedRequest(request=rewritten, stripped=stripped) + + +@dataclass(frozen=True, slots=True) +class StrippedNativeRequest: + thinking: object + thinking_stripped: bool + messages: Sequence[object] + messages_stripped: bool + stripped: tuple[str, ...] + + +def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest: + """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists.""" + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + return StrippedNativeRequest( + thinking=params.get("thinking"), + thinking_stripped=False, + messages=message_sequence if message_sequence is not None else (), + messages_stripped=False, + stripped=(), + ) + + +def strip_beta_gated_native_request( + headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object +) -> StrippedNativeRequest: + """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta`` + list on legs that carry betas in the body, plus the body parts it may rewrite.""" + if not drops_unsupported_params(drop_params): + return unstripped_native_request(optional_params, messages) + active_betas: Final = active_beta_values(headers, body_betas) + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + return StrippedNativeRequest( + thinking=thinking.thinking, + thinking_stripped=bool(thinking.stripped), + messages=system_messages.messages, + messages_stripped=bool(system_messages.stripped), + stripped=(*thinking.stripped, *system_messages.stripped), + ) + + +def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None: + if not stripped: + return + described: Final = ", ".join( + f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})" + if field in _FIELD_BETAS + else field + for field in sorted(set(stripped)) + ) + verbose_logger.warning( + "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on " + "this request, either since this provider leg does not send that anthropic-beta (see " + "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user " + "messages.", + described, + model, + provider, + ) + + +def strip_beta_gated_request_body( + request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str +) -> Mapping[str, object]: + gated: Final = strip_beta_gated_request_fields( + request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params + ) + log_beta_gated_strips(gated.stripped, model=model, provider=provider) + return gated.request diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index d68de6c9d4d..cee5ce15469 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -248,6 +248,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": null, "web-search-2025-03-05": "web-search-2025-03-05" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fec1d8e4465..07730a0fbf8 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str: return aliases.get(provider, provider) +def has_beta_headers_config(provider: str) -> bool: + config: Final = _bundled_beta_headers_config() + aliases: Final = config.get("provider_aliases", {}) + canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider + return canonical_provider in config + + def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, @@ -397,7 +404,7 @@ def update_request_with_filtered_beta( headers: dict, request_data: dict, provider: str, -) -> tuple[dict, dict]: +) -> tuple[dict[str, str], dict[str, object]]: """ Update both headers and request body beta fields based on provider support. Modifies both dicts in place and returns them. diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..1ced13445de 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -455,17 +455,33 @@ class Cache: 3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component) """ litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {} - metadata_sources: Final[tuple[dict, ...]] = ( - kwargs.get("metadata") or {}, - kwargs.get("litellm_metadata") or {}, - litellm_params.get("metadata") or {}, - litellm_params.get("litellm_metadata") or {}, + metadata_sources: Final = cast( # cast-ok: each source is the untyped request's metadata dict or empty + "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]", + ( + kwargs.get("metadata") or {}, + kwargs.get("litellm_metadata") or {}, + litellm_params.get("metadata") or {}, + litellm_params.get("litellm_metadata") or {}, + ), ) - model_group: Final[str | None] = next( - (source["model_group"] for source in metadata_sources if source.get("model_group")), None + metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources + call_model_info: Final[object] = kwargs.get("model_info") + routed_source: Final = next( + ( + source + for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata) + if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info + ), + None, ) + model_group: Final = ( + str(routed_source["model_group"]) + if routed_source is not None + else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None) + ) + caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources caching_group: Final = next( - (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None + (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None ) model_id_suffix: Final = ( f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else "" diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 924506dfa45..560a5ba06b8 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -13,7 +13,7 @@ import asyncio import json import os from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Protocol from pydantic import TypeAdapter @@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # create an embedding for prompt - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) @@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # convert to embedding - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index b892e4c3a1d..7268511fe7e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -14,7 +14,9 @@ import asyncio import json import os from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast + +from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import print_verbose, verbose_logger @@ -38,6 +40,11 @@ if TYPE_CHECKING: from litellm.router import Router +_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter( + str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + class RedisSemanticCache(BaseCache): """ Redis-backed semantic cache for LLM responses. @@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache): ) return embedding_response["data"][0]["embedding"] - def _get_cache_logic(self, cached_response: Any) -> object: + def _get_cache_logic(self, cached_response: str | bytes | None) -> object: """ Process the cached response to prepare it for use. @@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache): The processed cache response, or None if input was None """ if cached_response is None: - return cached_response + return None # Convert bytes to string if needed - if isinstance(cached_response, bytes): - cached_response = cached_response.decode("utf-8") + response_text: Final = ( + cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response + ) # Convert string representation to Python object try: - cached_response = json.loads(cached_response) + return json.loads(response_text) except json.JSONDecodeError: try: - cached_response = ast.literal_eval(cached_response) + return ast.literal_eval(response_text) except (ValueError, SyntaxError) as e: print_verbose(f"Error parsing cached response: {e}") return None - return cached_response - def set_cache(self, key: str, value: object, **kwargs) -> None: """ Store a value in the semantic cache. @@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity @@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3f7017eccc2..76d17ea9693 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replays_openai_history=custom_llm_provider == "openai", ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replays_openai_history=custom_llm_provider == "openai", ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/history.py b/litellm/completion_extras/litellm_responses_transformation/history.py new file mode 100644 index 00000000000..4435de64cde --- /dev/null +++ b/litellm/completion_extras/litellm_responses_transformation/history.py @@ -0,0 +1,432 @@ +import hashlib +import json +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import accumulate, chain, pairwise +from types import MappingProxyType +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +REPLAY_FIELD: Final = "openai_responses_replay" +PHASE_FIELD: Final = "phase" + +Phase: TypeAlias = Literal["commentary", "final_answer"] + + +class _Frozen(BaseModel): + model_config = ConfigDict(frozen=True) + + +class _OutputText(_Frozen): + type: Literal["output_text"] + text: str + + +class _OutputMessage(_Frozen): + type: Literal["message"] + role: Literal["assistant"] + content: tuple[_OutputText, ...] + phase: Phase | None = None + + +class _OutputReasoning(_Frozen): + type: Literal["reasoning"] + id: str + + +class _OutputToolCall(_Frozen): + type: Literal["function_call", "custom_tool_call"] + call_id: str + id: str | None = None + + +class _OutputToolSearch(_Frozen): + type: Literal["tool_search_call", "tool_search_output"] + + +_OutputItem: TypeAlias = Annotated[ + _OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type") +] +_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem) + + +class _AnyMessage(_Frozen): + type: Literal["message"] + + +class _MessageEntry(_Frozen): + type: Literal["message"] + parts: tuple[Annotated[int, Field(ge=0)], ...] + phase: Phase | None = None + + +class _IndexedEntry(_Frozen): + type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"] + index: Annotated[int, Field(ge=0)] + + +_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")] + + +class _Replay(_Frozen): + version: Literal[1] + items: tuple[_Entry, ...] + checksum: str + + +class _Namespaced(_Frozen): + namespace: str | None = None + + +class _ToolFunction(_Frozen): + name: str = "" + arguments: str = "" + provider_specific_fields: _Namespaced | None = None + + +class _ToolCall(_Frozen): + id: str + function: _ToolFunction + provider_specific_fields: _Namespaced | None = None + + +class _SummaryText(_Frozen): + type: str = "summary_text" + text: str = "" + + +class _Reasoning(_Frozen): + id: str | None = None + encrypted_content: str | None = None + summary: tuple[_SummaryText, ...] | None = None + + +class _ProviderFields(_Frozen): + phase: Phase | None = None + tool_search_items: tuple[Mapping[str, object], ...] | None = None + openai_responses_replay: object = None + + +class _ChatMessage(_Frozen): + role: str | None = None + content: str | None = None + tool_calls: tuple[_ToolCall, ...] | None = None + reasoning_items: tuple[_Reasoning, ...] | None = None + phase: Phase | None = None + provider_specific_fields: _ProviderFields | None = None + + +class _PhaseFields(_Frozen): + phase: Phase | None = None + + +class _RequestedPhaseFields(_Frozen): + phase: object = None + + +class _RequestedPhase(_Frozen): + phase: object = None + provider_specific_fields: _RequestedPhaseFields | None = None + + +@dataclass(frozen=True, slots=True) +class UnknownPhase: + value: object + + +_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase) + + +class _RetainedFields(_PhaseFields): + openai_responses_replay: object = None + + +@dataclass(frozen=True, slots=True) +class ReplayedTurn: + items: tuple[Mapping[str, object], ...] + custom_call_ids: frozenset[str] + + +def _as_json(value: object) -> object: + return value.model_dump() if isinstance(value, BaseModel) else value + + +def _parsed_output_item(item: object) -> _OutputItem | None: + try: + return _OUTPUT_ITEM.validate_python(_as_json(item)) + except ValidationError: + return None + + +def _is_message(item: object) -> bool: + try: + _AnyMessage.model_validate(_as_json(item)) + except ValidationError: + return False + return True + + +def _chat_message(message: object) -> _ChatMessage | None: + try: + return _ChatMessage.model_validate(_as_json(message)) + except ValidationError: + return None + + +def requested_phase(message: object) -> Phase | UnknownPhase | None: + try: + requested: Final = _RequestedPhase.model_validate(_as_json(message)) + except ValidationError: + return None + fields: Final = requested.provider_specific_fields + value: Final = requested.phase or (fields.phase if fields else None) + if value is None: + return None + try: + return _PHASE.validate_python(value) + except ValidationError: + return UnknownPhase(value) + + +def reasoning_input_item( + item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None +) -> Mapping[str, object]: + return { + "type": "reasoning", + **({"id": item_id} if item_id else {}), + "summary": list(summary or ()), + **({"encrypted_content": encrypted_content} if encrypted_content else {}), + } + + +def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None: + try: + retained: Final = _RetainedFields.model_validate(provider_specific_fields) + except ValidationError: + return None + return retained.model_dump(exclude_none=True) or None + + +def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str: + parsed: Final = map(_parsed_output_item, output_items) + messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage)) + return "".join(_texts(messages[:1] if first_message_only else messages)) + + +def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]: + fields: Final = message.provider_specific_fields + items: Final = (fields.tool_search_items if fields else None) or () + return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output")) + + +def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str: + fields: Final = message.provider_specific_fields + bound: Final = { + "content": message.content or "", + "tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()], + "reasoning": [ + [item.id, item.encrypted_content, [part.text for part in item.summary or ()]] + for item in message.reasoning_items or () + ], + "tool_search_items": list((fields.tool_search_items if fields else None) or ()), + "phase": message.phase or (fields.phase if fields else None), + "items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries], + } + encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest() + + +def _kind(item: _OutputItem) -> str: + match item: + case _OutputToolCall(): + return "tool_call" + case _OutputToolSearch(): + return "tool_search" + case _OutputMessage() | _OutputReasoning(): + return item.type + + +def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry: + item: Final = items[position] + if isinstance(item, _OutputMessage): + return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase) + ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item)) + return _IndexedEntry(type=item.type, index=ordinal) + + +def _texts(items: Sequence[_OutputItem]) -> Iterator[str]: + for item in items: + if isinstance(item, _OutputMessage): + yield from (part.text for part in item.content) + + +def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool: + reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning)) + calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall)) + tool_calls: Final = message.tool_calls or () + searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch)) + return ( + "".join(_texts(items)) == (message.content or "") + and reasoning_ids == tuple(item.id for item in message.reasoning_items or ()) + and len(calls) == len(tool_calls) + and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls)) + and searches == len(_tool_search_entries(message)) + ) + + +def _content_phase( + output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool +) -> Phase | None: + messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw)) + if not messages: + return None + content_message: Final = messages[0] if first_message_only else messages[-1] + return content_message.phase if isinstance(content_message, _OutputMessage) else None + + +def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage: + if phase is None: + return message + fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase}) + return message.model_copy(update={"provider_specific_fields": fields}) + + +def replay_fields( + output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool +) -> Mapping[str, object]: + parsed: Final = tuple(map(_parsed_output_item, output_items)) + phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only) + phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {}) + items: Final = tuple(item for item in parsed if item is not None) + parsed_message: Final = _chat_message(message) + chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None + if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat): + return phase_fields + entries: Final = tuple(_entry(items, position) for position in range(len(items))) + replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries)) + return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)}) + + +def _stored_replay(message: _ChatMessage) -> _Replay | None: + fields: Final = message.provider_specific_fields + if fields is None or fields.openai_responses_replay is None: + return None + try: + return _Replay.model_validate(fields.openai_responses_replay) + except ValidationError: + return None + + +def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]: + lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries)) + bounds: Final = tuple(accumulate(lengths, initial=0)) + parts: Final = tuple(content[start:end] for start, end in pairwise(bounds)) + groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0)) + return tuple(parts[start:end] for start, end in pairwise(groups)) + + +def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]: + return { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": text} for text in texts], + **({PHASE_FIELD: phase} if phase else {}), + } + + +def _function_call_item(call: _ToolCall) -> Mapping[str, object]: + namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields + namespace: Final = namespaced.namespace if namespaced else None + return { + "type": "function_call", + "call_id": call.id, + "name": call.function.name, + "arguments": call.function.arguments, + **({"namespace": namespace} if namespace else {}), + } + + +def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]: + return { + "type": "custom_tool_call", + "call_id": call.id, + "name": call.function.name, + "input": call.function.arguments, + } + + +def _reasoning_item(item: _Reasoning) -> Mapping[str, object]: + summary: Final = tuple(part.model_dump() for part in item.summary or ()) + return reasoning_input_item(item.id, summary, item.encrypted_content) + + +@dataclass(frozen=True, slots=True) +class _Resolved: + message: _ChatMessage + texts: Mapping[int, tuple[str, ...]] + tool_searches: tuple[Mapping[str, object], ...] + + +def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]: + if isinstance(entry, _MessageEntry): + return _message_item(resolved.texts[position], entry.phase) + tool_calls: Final = resolved.message.tool_calls or () + match entry.type: + case "reasoning": + return _reasoning_item((resolved.message.reasoning_items or ())[entry.index]) + case "function_call": + return _function_call_item(tool_calls[entry.index]) + case "custom_tool_call": + return _custom_tool_call_item(tool_calls[entry.index]) + case "tool_search_call" | "tool_search_output": + return dict(resolved.tool_searches[entry.index]) + + +def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool: + tool_calls: Final = len(message.tool_calls or ()) + tool_searches: Final = len(_tool_search_entries(message)) + sizes: Final = MappingProxyType( + { + "reasoning": len(message.reasoning_items or ()), + "function_call": tool_calls, + "custom_tool_call": tool_calls, + "tool_search_call": tool_searches, + "tool_search_output": tool_searches, + } + ) + parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry)) + indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry)) + return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed) + + +def _resolve( + message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool +) -> ReplayedTurn | None: + indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry)) + if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed): + return None + messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry)) + calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call")) + texts: Final = _message_texts(messages, message.content or "") + positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry)) + resolved: Final = _Resolved( + message=message, + texts=MappingProxyType(dict(zip(positions, texts, strict=True))), + tool_searches=_tool_search_entries(message), + ) + tool_calls: Final = message.tool_calls or () + return ReplayedTurn( + items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)), + custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"), + ) + + +def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None: + chat: Final = _chat_message(message) + stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None + if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items): + return None + if not _fits(chat, stored.items): + return None + return _resolve(chat, stored.items, replay_tool_search) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index ee3639ac387..580e55273f6 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm import ModelResponse from litellm._logging import verbose_logger +from litellm.completion_extras.litellm_responses_transformation.history import ( + PHASE_FIELD, + UnknownPhase, + output_message_text, + reasoning_input_item, + replay_fields, + replay_items, + requested_phase, +) from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) @@ -52,7 +62,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, - PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -490,7 +499,7 @@ def _fold_choices_into_one( choices: Sequence["Choices"], tool_calls: Sequence[Mapping[str, object]], trailing_reasoning_content: str | None, - trailing_reasoning_item: _BuiltReasoningItem | None, + trailing_reasoning_items: Sequence[_BuiltReasoningItem], tool_search_items: Sequence[Mapping[str, object]], structured_output: bool = False, ) -> "Choices": @@ -509,7 +518,7 @@ def _fold_choices_into_one( reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content) reasoning_items: Final = ( *chain.from_iterable(_message_reasoning_items(m) for m in messages), - *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), + *trailing_reasoning_items, ) return Choices( message=Message( @@ -526,6 +535,61 @@ def _fold_choices_into_one( ) +def _with_replay_fields( + choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool +) -> "Choices": + fields: Final = { + **(choice.message.provider_specific_fields or {}), + **replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output), + } + message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None}) + return choice.model_copy(update={"message": message}) + + +_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) + + +def completed_stream_history( + output_payload: object, + tool_search_fields: Mapping[str, object] | None, + *, + completed: bool, + structured_output: bool, +) -> dict[str, object] | None: # mutable-ok: Delta.provider_specific_fields is typed as a dict + items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload) + calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call")) + streamed_message: Final = { + "role": "assistant", + "content": output_message_text(items, first_message_only=structured_output), + "tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)], + "reasoning_items": list(_reasoning_items_from_output_items(items)), + "provider_specific_fields": tool_search_fields, + } + fields: Final = { + **(tool_search_fields or {}), + **replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output), + } + return fields or None + + +def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]: + match requested_phase(msg) if replay_openai_history else None: + case None: + return {} + case UnknownPhase(value=value) if drop_params: + verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value) + return {} + case UnknownPhase(value=value): + raise litellm.UnsupportedParamsError( + message=f"phase must be 'commentary' or 'final_answer', got {value!r}. " + "To drop it, set `litellm.drop_params=True` or pass `drop_params=True`." + ) + case phase: + return {PHASE_FIELD: phase} + + def _responses_event_type(chunk: Mapping[str, object]) -> object: event_type: Final = chunk.get("type") return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type @@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) - - -def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - try: - return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - - def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -667,15 +718,7 @@ def _reasoning_item_to_response_input( An item without an id is sent without one: the Responses API accepts that and verifies the encrypted content on its own, while it rejects any id it did not mint. """ - item_id: Final = r_item.get("id") - encrypted_content: Final = r_item.get("encrypted_content") - return { - "type": "reasoning", - **({"id": item_id} if item_id else {}), - # summary is always required by the Responses API, even when empty - "summary": r_item.get("summary") or [], - **({"encrypted_content": encrypted_content} if encrypted_content else {}), - } + return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content"))) def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object: @@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_openai_history: bool = False, ) -> tuple[list[object], str | None]: converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input( messages, drop_params=drop_params, replay_tool_search=replay_tool_search, include_namespace=include_namespace, + replay_openai_history=replay_openai_history, ) return ( converted_input_items @@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): drop_params: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_openai_history: bool = False, ) -> tuple[list[object], str | None]: input_items: Final[list[object]] = [] instructions: str | None = None - custom_tool_call_ids: Final = frozenset( + replayed_turns: Final = tuple( + replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None + for msg in messages + ) + replayed_custom_call_ids: Final = frozenset( + chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None) + ) + custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset( tool_call["id"] for msg in messages if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list) @@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_calls = msg.get("tool_calls") tool_call_id = msg.get("tool_call_id") - if role == "system": + if (replayed_turn := replayed_turns[index]) is not None: + input_items.extend(replayed_turn.items) + elif role == "system": # Extract system message as instructions if isinstance(content, str) and index < leading_system_count: if instructions: @@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, "assistant", drop_params=drop_params ), + **_assistant_phase_field(msg, replay_openai_history, drop_params), } ) for tool_call in tool_calls: @@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, cast(str, role), drop_params=drop_params ), + **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params), } ) elif role == "assistant": @@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, + replay_openai_history=provider == "openai", ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead @@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items: Sequence[object], handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None, structured_output: bool = False, + replay_openai_history: bool = False, + completed: bool = False, ) -> list["Choices"]: """ Convert Responses API output items to chat completion choices. @@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): handle_raw_dict_callback: Optional callback for handling raw dict items structured_output: The chat request carried a json response_format, so only the first message item becomes content + replay_openai_history: Record the output's item sequence for the next turn + completed: The response finished, so its item sequence is whole Returns: List of Choices objects @@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices: Final[list[Choices]] = [] index = 0 reasoning_content: str | None = None - pending_reasoning_item: _BuiltReasoningItem | None = None + pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = () # Collect all tool calls to put them in a single choice # (Chat Completions API expects all tool calls in one message) @@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - pending_reasoning_item = _build_reasoning_item( - item_id=item.id, - encrypted_content=getattr(item, "encrypted_content", None), - summary_raw=item.summary, + pending_reasoning_items = ( + *(pending_reasoning_items if replay_openai_history else ()), + _build_reasoning_item( + item_id=item.id, + encrypted_content=getattr(item, "encrypted_content", None), + summary_raw=item.summary, + ), + ) + reasoning_content = " ".join( + s["text"] + for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items) + if s.get("text") ) - reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text")) elif isinstance(item, ResponseOutputMessage): for content in item.content: @@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotations=annotations, reasoning_items=cast( list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), + (list(pending_reasoning_items) or None), ), ) @@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) reasoning_content = None # flush - pending_reasoning_item = None # flush + pending_reasoning_items = () # flush index += 1 elif isinstance(item, ResponseFunctionToolCall): @@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items, "stop", tool_search_items=tool_search_items ) ] - return [ - _fold_choices_into_one( - choices=choices, - tool_calls=accumulated_tool_calls, - trailing_reasoning_content=reasoning_content, - trailing_reasoning_item=pending_reasoning_item, - tool_search_items=tool_search_items, - structured_output=structured_output, - ) - ] + folded: Final = _fold_choices_into_one( + choices=choices, + tool_calls=accumulated_tool_calls, + trailing_reasoning_content=reasoning_content, + trailing_reasoning_items=pending_reasoning_items, + tool_search_items=tool_search_items, + structured_output=structured_output, + ) + if not replay_openai_history: + return [folded] + return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)] @staticmethod def _build_empty_choice( @@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + replay_openai_history=_served_provider( + cast("Mapping[str, object]", litellm_params) # cast-ok: litellm_params is an untyped dict + ) + == "openai", + completed=raw_response.status == "completed", ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( @@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + replays_openai_history: bool = False, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( - streaming_response, sync_stream, json_mode, structured_output=structured_output + streaming_response, + sync_stream, + json_mode, + structured_output=structured_output, + replays_openai_history=replays_openai_history, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) elif original_type == "input_audio": converted = with_prompt_cache_breakpoint( {"type": "input_audio", "input_audio": item.get("input_audio")}, - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: input_audio -> %s", converted) @@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) @@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + replays_openai_history: bool = False, ): super().__init__(streaming_response, sync_stream, json_mode) + self._replays_openai_history: Final = replays_openai_history self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state @@ -1937,6 +2019,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def translate_responses_chunk_to_openai_stream( parsed_chunk: dict | BaseModel, tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place + replays_openai_history: bool = False, + structured_output: bool = False, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -1944,6 +2028,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): Args: parsed_chunk: Dict containing the Responses API event chunk tool_call_index_map: Per-stream output_index -> sequential tool_call index map + replays_openai_history: Record the completed output's item sequence for the next turn + structured_output: Only the first message item became content Returns: ModelResponseStream: OpenAI-formatted streaming chunk @@ -2154,6 +2240,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): terminal_reasoning_items_typed: Final = _as_chat_reasoning_items( _reasoning_items_from_output_items(output_items) ) + output_payload: Final = cast(object, output_items) # cast-ok: untyped + tool_search_fields: Final = _tool_search_provider_fields( + _tool_search_items_from_output_items(output_payload) + ) usage = None if response_data.get("usage"): @@ -2167,10 +2257,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): StreamingChoices( index=0, delta=Delta( - content="", + content=None if replays_openai_history else "", reasoning_items=terminal_reasoning_items_typed, - provider_specific_fields=_tool_search_provider_fields( - _tool_search_items_from_output_items(cast(object, output_items)) # cast-ok: untyped + provider_specific_fields=( + completed_stream_history( + output_payload, + tool_search_fields, + completed=event_type == "response.completed", + structured_output=structured_output, + ) + if replays_openai_history + else tool_search_fields ), ), finish_reason=finish_reason, @@ -2216,7 +2313,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( - chunk, tool_call_index_map=self._tool_call_index_map + chunk, + tool_call_index_map=self._tool_call_index_map, + replays_openai_history=self._replays_openai_history, + structured_output=self._structured_output, ) ) if self._structured_output and _is_message_item_done(chunk): diff --git a/litellm/constants.py b/litellm/constants.py index 42ad624026c..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_ DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range( - "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400 + "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400 ) # Maximum wall-clock seconds a streaming response is allowed to run. @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5dc0ce13de..a4a9a5bee39 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_prompt_tokens_details, select_cost_metric_for_model, ) +from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -1396,9 +1397,9 @@ def _apply_cost_margin( def _store_cost_breakdown_in_logging_obj( litellm_logging_obj: LitellmLoggingObject | None, - prompt_tokens_cost_usd_dollar: float, - completion_tokens_cost_usd_dollar: float, - cost_for_built_in_tools_cost_usd_dollar: float, + prompt_tokens_cost_usd_dollar: float | None, + completion_tokens_cost_usd_dollar: float | None, + cost_for_built_in_tools_cost_usd_dollar: float | None, total_cost_usd_dollar: float, additional_costs: dict | None = None, original_cost: float | None = None, @@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost( return float(stamped) if total_time: return total_time - if litellm_logging_obj is not None: - return litellm_logging_obj.get_response_ms() - return 0.0 + if litellm_logging_obj is None: + return 0.0 + reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj) + if reported_ms is not None and litellm_logging_obj.stream is not True: + return reported_ms + return litellm_logging_obj.get_response_ms() + + +def _adjusted_provider_billed_cost( + reported_cost: float, + custom_llm_provider: str | None, + model: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, + service_tier: str | None, + data_residency: str | None, +) -> float: + discounted_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin( + base_cost=discounted_cost, custom_llm_provider=custom_llm_provider + ) + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=None, + completion_tokens_cost_usd_dollar=None, + cost_for_built_in_tools_cost_usd_dollar=None, + total_cost_usd_dollar=adjusted_cost, + original_cost=reported_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + return adjusted_cost def completion_cost( @@ -1616,6 +1653,9 @@ def completion_cost( ) explicit_pricing: Final = custom_pricing is True or base_model is not None + caller_set_prices: Final = ( + explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None @@ -1666,6 +1706,7 @@ def completion_cost( raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) + priced_response: Final[object] = completion_response for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names): model = candidate_model try: @@ -1984,6 +2025,19 @@ def completion_cost( total_time=total_time, ) model_for_discount = custom_priced_request_model or model + if ( + not caller_set_prices + and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None + ): + return _adjusted_provider_billed_cost( + reported_cost=reported_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" diff --git a/litellm/images/main.py b/litellm/images/main.py index 6e77cbb7046..b3247ff938b 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -641,6 +641,8 @@ def image_generation( timeout=timeout, client=custom_client, ) + elif custom_llm_provider == LlmProviders.KYMA.value: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return model_response except Exception as e: diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..fc1f1830129 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") +_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None: def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True + + +def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: hosted_flag: Final = ( None if custom_llm_provider is None @@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: return model_map_flag - version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) - if version_match is None: + fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) + if fine_tuned_base is not None: + return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None + return _named_model_breakpoint_support(model) + + +def _named_model_breakpoint_support(model: str) -> bool | None: + bare_model: Final = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_PATTERN.match(bare_model) is not None: return False + version_match: Final = _GPT_VERSION_PATTERN.match(bare_model) + if version_match is None: + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a4de1be1842..0579eb15baf 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -513,7 +513,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac data: dict, user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, - ) -> Any: + ) -> object: pass async def async_post_call_streaming_hook( diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 1a81fa019dc..d4b75fc0bce 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger): # see _RESERVED_TAG_KEYS for the rationale. if self.cost_tag_keys: allow: Final = set(self.cost_tag_keys) + client_allow: Final = allow - {"is_silent_experiment"} for rt in log.get("request_tags") or []: if not isinstance(rt, str) or ":" not in rt: continue k, _, v = rt.partition(":") - if k in allow and v: + if k in client_allow and v: self._set_custom_tag(tags, k, v) for k, v in metadata.items(): if k in allow and v is not None and not isinstance(v, (dict, list)): @@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): nested = metadata.get(nested_key) if isinstance(nested, dict): for k, v in nested.items(): - if k in allow and v is not None and not isinstance(v, (dict, list)): + if k in client_allow and v is not None and not isinstance(v, (dict, list)): self._set_custom_tag(tags, k, str(v)) return tags diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..bd68ffe8cd6 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -833,6 +833,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py index 32cf93b7937..97a970c8966 100644 --- a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py +++ b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py @@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC): pass -def cast_as_primitive_value_type(value) -> str | bool | int | float: +def cast_as_primitive_value_type(value: object) -> str | bool | int | float: """ Converts a value to an OTEL-supported primitive for Arize/Phoenix observability. """ diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index bc8344fa686..e62b99f8b0f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1173,8 +1173,8 @@ class PrometheusLogger(CustomLogger): # Build label filters from valid configurations label_filters: Final = self._build_label_filters(parsed_configs) - # Pretty print the processed configuration - self._pretty_print_prometheus_config(label_filters) + if litellm.suppress_startup_output is not True: + self._pretty_print_prometheus_config(label_filters) return label_filters def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]: @@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload( **(requester_metadata if isinstance(requester_metadata, dict) else {}), **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}), **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), + "is_silent_experiment": ( + "true" + if "is_silent_experiment" in standard_logging_metadata + and standard_logging_metadata["is_silent_experiment"] is True + else None + ), } diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..c25fcfe3e70 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict): websearch_interception_params: WebSearchInterceptionConfig +def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str: + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {} + } + recorded_model: Final = agentic_view["agentic_loop_params"].get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + return qualify_provider_stripped_model( + recorded_model if isinstance(recorded_model, str) and recorded_model else model, + custom_llm_provider if isinstance(custom_llm_provider, str) else "", + ) + + class _SearchToolLitellmParams(TypedDict, total=False): search_provider: ReadOnly[str | None] @@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, response_format=response_format, ) return AgenticLoopPlan( @@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, ) return AgenticLoopPlan( run_agentic_loop=True, @@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: list[dict], optional_params: dict, kwargs: dict, + logging_obj: "LiteLLMLoggingObj | None", ) -> AgenticLoopRequestPatch: """Execute litellm.asearch() and build a Responses API rerun patch.""" search_tasks: Final = [ @@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=dict[str, object](optional_params), kwargs=dict[str, object](kwargs), + logging_obj=logging_obj, response_format=response_format, ) if request_patch.messages is None: @@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], + logging_obj: "LiteLLMLoggingObj | None", response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index d84f0b5ffbd..f95b5611831 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=custom_llm_provider, ) async def async_interactions_api_handler( @@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=responses_request.get("custom_llm_provider"), ) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index ade33831cdc..2e9b62dcfe2 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -12,6 +12,7 @@ from typing import Final, cast from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import billed_usage_cost from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig: def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, model: str | None = None, + custom_llm_provider: str | None = None, ) -> InteractionsAPIResponse: """ Transform a Responses API response to an Interactions API response. @@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: + reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), + **({} if reported_cost is None else {"cost": reported_cost}), } # Add updated (same as created for now) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 8a61ebff25f..722afe06d9b 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -35,6 +35,49 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider + + +def _leads_with_provider(model: str) -> bool: + leading_segment, separator, _ = model.partition("/") + return bool(separator) and names_a_provider(leading_segment) + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + if _leads_with_provider(patch_model): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. @@ -985,6 +1028,16 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) +USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) + + +def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: + if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + return None + cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None + + _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47ec8331c83..703550dd2b5 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -2762,6 +2763,8 @@ def exception_type( exception_provider=exception_provider, extra_information=extra_information, ) + if custom_llm_provider == "bigmodel": + raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception) # pyright: ignore[reportUnknownArgumentType] # exception_type takes untyped parameters if custom_llm_provider == "openrouter": _map_openrouter_exception( model=model, diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index d9933087334..f8c8ff71f8e 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "bigmodel": + from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url + + ( + api_base, # rebind-ok: existing provider-resolution pipeline resolves the API base in place + dynamic_api_key, # rebind-ok: existing provider-resolution pipeline resolves the API key in place + ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) elif custom_llm_provider == "zai": ( api_base, diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index 1200d799018..f3dbd612bee 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -120,7 +120,7 @@ def is_unbilled_non_inference_call( origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: return False - if not is_background_response(response): + if not is_background_response(_finished_response(response)): return True return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 516e9cce448..f8f35bc9dc3 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -23,7 +23,7 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import post_response_phase +from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase from litellm._logging import ( is_debugging_on, redact_string, @@ -164,9 +164,10 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, + AgenticLoopParams, CachingDetails, CallTypes, CostBreakdown, @@ -316,7 +317,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -810,6 +811,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment: Final = in_mirrored_call() # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1160,6 +1162,9 @@ class Logging(LiteLLMLoggingBaseClass): if "custom_llm_provider" in self.model_call_details: self.custom_llm_provider = self.model_call_details["custom_llm_provider"] + def record_agentic_loop_params(self, params: AgenticLoopParams) -> None: + self.model_call_details["agentic_loop_params"] = params + def record_applied_timeout(self, timeout: float | Timeout) -> None: self.applied_timeout = timeout self.litellm_params["timeout"] = timeout @@ -1939,10 +1944,10 @@ class Logging(LiteLLMLoggingBaseClass): def set_cost_breakdown( self, - input_cost: float, - output_cost: float, + input_cost: float | None, + output_cost: float | None, total_cost: float, - cost_for_built_in_tools_cost_usd_dollar: float, + cost_for_built_in_tools_cost_usd_dollar: float | None, additional_costs: dict | None = None, original_cost: float | None = None, discount_percent: float | None = None, @@ -1981,14 +1986,17 @@ class Logging(LiteLLMLoggingBaseClass): self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( - input_cost=input_cost, - output_cost=output_cost, total_cost=total_cost, - tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, ) + if input_cost is not None: + self.cost_breakdown["input_cost"] = input_cost + if output_cost is not None: + self.cost_breakdown["output_cost"] = output_cost + if cost_for_built_in_tools_cost_usd_dollar is not None: + self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost if cache_creation_cost is not None and cache_creation_cost > 0: @@ -3260,7 +3268,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return self._success_handler_body( result=result, start_time=start_time, @@ -3760,7 +3768,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return await self._async_success_handler_body( result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs ) @@ -4277,13 +4285,14 @@ class Logging(LiteLLMLoggingBaseClass): ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: - return self._failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - for_async_request=for_async_request, - ) + with outside_mirrored_call(): + return self._failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + for_async_request=for_async_request, + ) finally: self._restore_correlation_context() @@ -4476,12 +4485,13 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own failure logging (including any nested calls its callbacks trigger) is fully done.""" try: - return await self._async_failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - ) + with outside_mirrored_call(): + return await self._async_failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + ) finally: self._restore_correlation_context() @@ -6570,6 +6580,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6643,6 +6654,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7443,6 +7455,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7735,9 +7748,8 @@ def get_standard_logging_metadata( ) if isinstance(metadata, dict): # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields - for key in StandardLoggingMetadata.__annotations__: - if key in metadata: - clean_metadata[key] = metadata[key] + for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}: + clean_metadata[key] = metadata[key] if metadata.get("user_api_key") is not None: if is_valid_sha256_hash(str(metadata.get("user_api_key"))): diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 0bf59a6d1f2..db881aa16c6 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,8 +1,9 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import Final, cast import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider def _form_field_value(value: object) -> str: @@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): if custom_llm_provider not in litellm.models_by_provider: return [] - known_models: Final = litellm.models_by_provider.get(custom_llm_provider, []) - model_costs: Final = [] + known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) + ranked_rows: Final = ( + (route_rank, cost, model) + for model, cost in _chat_row_costs(known_models, custom_llm_provider) + if (route_rank := _route_rank(model, custom_llm_provider)) is not None + ) + return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]] - for model in known_models: + +def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: + for model in models: try: model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: continue if model_info.get("mode") != "chat": continue - _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - model_costs.append((model, _cost)) + yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - # Sort by cost (ascending) - model_costs.sort(key=lambda x: x[1]) - # Return the top n cheapest models - return [model for model, _ in model_costs[:n]] +def _route_rank(model: str, custom_llm_provider: str) -> int | None: + routed_provider: Final = inferred_provider(model) + if routed_provider is None: + return 1 + if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"): + return 0 + return None def get_proxy_server_request_headers(litellm_params: dict | None) -> dict: diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..051a8fdf256 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -27,6 +27,15 @@ def _timing_window_start( return start_time, False +def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None: + reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms") + if reported_ms is None: + return None + logging_start: Final = logging_obj.start_time + window_start: Final = _timing_window_start(logging_start, logging_obj)[0] + return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000 + + def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None: if not isinstance(windows, (list, tuple)): return None diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 101f069c0f8..9bc53791b36 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -17,7 +17,7 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger @@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" if block.get("cache_control") is None: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8be7a056166..bc826c8e74d 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,12 +12,18 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import chunk_usage, finite_cost, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + USAGE_COST_BILLED_PROVIDERS, + chunk_usage, + finite_cost, + provider_reported_usage_cost, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) +from litellm.litellm_core_utils.token_counter import usage_tokenizer_model from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantContentValue, @@ -51,7 +57,7 @@ from litellm.types.utils import ( UnreportedUsage, Usage, ) -from litellm.utils import print_verbose, token_counter +from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter if TYPE_CHECKING: from openai.types.completion_usage import CompletionUsage @@ -1353,7 +1359,7 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS def _logged_provider(logging_obj: Optional["Logging"]) -> object: @@ -1554,6 +1560,9 @@ def stream_chunk_builder( return stream_chunk_builder_text_completion(chunks=chunks, messages=messages) model: Final = chunks[0]["model"] + usage_model: Final = usage_tokenizer_model( + model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None + ) # Initialize the response dictionary response: Final = processor.build_base_response(chunks) @@ -1577,7 +1586,7 @@ def stream_chunk_builder( completion_output = get_content_from_model_response(response) usage = processor.calculate_usage( chunks=chunks, - model=model, + model=usage_model, completion_output=completion_output, messages=messages, reasoning_tokens=0, @@ -1768,7 +1777,7 @@ def stream_chunk_builder( usage = processor.calculate_usage( chunks=chunks, - model=model, + model=usage_model, completion_output=completion_output, messages=messages, reasoning_tokens=reasoning_tokens, diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f2f06d801a8..5542249d2c7 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1938,6 +1938,16 @@ class CustomStreamWrapper: model_response: Final = self.model_response_creator( hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id} ) + served_tier: Final = next( + ( + tier + for chunk in reversed(self.chunks) + if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier + ), + None, + ) + if served_tier is not None: + setattr(model_response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason if _finish_reason is not None: model_response.choices[0].finish_reason = _finish_reason diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index f42e4eae08f..591d77bdb16 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -15,6 +15,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar import litellm @@ -49,6 +50,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value +from litellm.router_utils.deployment_pricing import deployment_pricing_models from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, @@ -609,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( - ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") + ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom") ) @@ -1005,17 +1007,144 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: - """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose + """The tiktoken encoding name for `model`, without loading the encoding. - model_to_use: Final = _fix_model_name(model) + A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does + know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every + OpenAI model since gpt-4o uses. Provider and region prefixes (`azure/eu/`, `azure_ai/`) are dropped, a name + listed only under Azure (`gpt-41`, `openai/o1-mini`) counts as listed through its `azure/` cost-map key, and an + unlisted name tiktoken counts with o200k_base (`openai/gpt-5.9`) or named like a gpt-5 or o-series release + (`openai/gpt-6.2`) gets it.""" + model_to_use: Final = _openai_model_name(model) or _fix_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" + + +def unresolved_tokenizer_model(model: str) -> str: + """`model`, or `gpt-3.5-turbo` when resolving the OpenAI model `model` names moves it off cl100k_base, the encoding + tiktoken gives the name as written, so a count with the result never drops below that one.""" + as_written: Final = _fix_model_name(model) + as_written_encoding: Final = ( + "o200k_base" if "gpt-4o" in as_written else (_tiktoken_encoding_name(as_written) or "cl100k_base") + ) + moved_off_cl100k: Final = ( + as_written_encoding == "cl100k_base" and openai_tokenizer_encoding_name(model) != "cl100k_base" + ) + return "gpt-3.5-turbo" if moved_off_cl100k else model + + +def usage_tokenizer_model(model: str, base_model: str | None) -> str: + """The model to estimate missing usage with: a deployment's `base_model` when it names an OpenAI model, since a + custom deployment name says nothing about its tokenizer, else `model`.""" + return base_model if base_model is not None and is_openai_model(base_model) else model + + +def is_openai_model(model: str) -> bool: + """Whether `model` names an OpenAI or Azure OpenAI model, listed in the cost map or counted with o200k_base by + tiktoken, with or without its prefixes.""" + return _openai_model_name(model) is not None + + +def deployment_openai_model(deployment: object) -> str | None: + """The OpenAI model a router deployment serves, read from its base model first, since a custom deployment name + (`azure/my-gpt-5`) says nothing about its tokenizer, or None when it serves no OpenAI model.""" + return _as_openai_model(_served_model(deployment)) + + +def served_openai_models(deployments: Iterable[object]) -> tuple[str | None, ...]: + """The distinct OpenAI models `deployments` serve, with None standing for any deployment serving another model.""" + served: Final = dict.fromkeys(_served_model(deployment) for deployment in deployments) + return tuple(dict.fromkeys(_as_openai_model(model) for model in served)) + + +def _served_model(deployment: object) -> str | None: + return next(iter(deployment_pricing_models(deployment)), None) + + +def _as_openai_model(model: str | None) -> str | None: + return model if model is not None and is_openai_model(model) else None + + +def single_encoding_openai_model(served: Sequence[str | None]) -> str | None: + """An OpenAI model to count a request with, when every `served_openai_models` entry is an OpenAI model on the same + encoding, else None.""" + if not served or None in served: + return None + encodings: Final = frozenset(openai_tokenizer_encoding_name(model) for model in served if model is not None) + return served[0] if len(encodings) == 1 else None + + +def _openai_model_name(model: str) -> str | None: + return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model) + + +_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)") + + +def _versioned_openai_model_name(model: str) -> str | None: + name: Final = model.rsplit("/", 1)[-1] + return name if _VERSIONED_OPENAI_MODEL.match(name) else None + + +def _tiktoken_o200k_model_name(model: str) -> str | None: + name: Final = model.rsplit("/", 1)[-1] + return name if _tiktoken_encoding_name(name) == "o200k_base" else None + + +def _listed_openai_model_name(model: str) -> str | None: + segments: Final = model.split("/") + suffixes: Final = ("/".join(segments[start:]) for start in range(len(segments))) + listed: Final = any( + _is_openai_cost_map_model(suffix) or _is_openai_cost_map_model(f"azure/{suffix}") for suffix in suffixes + ) + return segments[-1] if listed else None + + +def _is_openai_cost_map_model(model: str) -> bool: + return ( + model in litellm.azure_llms + or model in litellm.open_ai_chat_completion_models + or _is_azure_openai_text_model(model) + ) + + +_TEXT_MODEL_MODES: Final = frozenset({"chat", "completion", "responses", "realtime"}) + + +def _is_azure_openai_text_model(model: str) -> bool: + cost_info: Final[Mapping[str, object]] = litellm.model_cost.get(model) or {} + return ( + cost_info.get("litellm_provider") == "azure" + and cost_info.get("mode") in _TEXT_MODEL_MODES + and _is_openai_model_name(model.rsplit("/", 1)[-1]) + ) + + +def _is_openai_model_name(name: str) -> bool: + return ( + name in litellm.open_ai_chat_completion_models + or _tiktoken_encoding_name(name) is not None + or any( + name.startswith(f"{listed}-") or listed.startswith(f"{name}-") + for listed in tuple(litellm.open_ai_chat_completion_models) + ) + ) + + +def _tiktoken_encoding_name(model: str) -> str | None: + try: + return tiktoken.encoding_name_for_model(model) + except KeyError: + return None + + +def _suffixed_variant_encoding_name(model: str) -> str | None: + known: Final = tuple(name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")) + return MODEL_TO_ENCODING[max(known, key=len)] if known else None def uses_legacy_message_accounting(model: str) -> bool: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index d371aab4581..31b0f06251d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation): message, _ = serialize_http_exception_detail(exc.detail) return tuple(anthropic_sse_error_frames(message)) - def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse": import uuid - from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( - FakeAnthropicMessagesStreamIterator, - ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_response_usage, ) from litellm.types.utils import AnthropicMessagesResponse - block_response: Final = AnthropicMessagesResponse( + return AnthropicMessagesResponse( id=f"msg_{uuid.uuid4()}", type="message", role="assistant", content=[{"type": "text", "text": exc.message}], model=exc.model, stop_reason="end_turn", - usage=blocked_response_usage(getattr(exc, "original_response", None)), + usage=blocked_response_usage(exc.original_response), ) - return list(FakeAnthropicMessagesStreamIterator(response=block_response)) + + def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc))) def _block_continuation_chunks( self, exc: "ModifyResponseException", responses_so_far: Sequence[object] diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 8a82ed98991..e1436473426 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -17,6 +17,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import ( update_request_with_filtered_beta, ) @@ -506,11 +507,20 @@ class AnthropicChatCompletion(BaseLLM): The pair stays mutable because the streaming path rewrites it in place (`data["stream"] = True`) before sending. """ - request_headers, data = update_request_with_filtered_beta( + request_headers, filtered_data = update_request_with_filtered_beta( headers=headers, request_data=request_data, provider=custom_llm_provider, ) + data: Final = dict( + strip_beta_gated_request_body( + filtered_data, + request_headers, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) logging_obj.pre_call( input=messages, api_key=api_key, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..3b7fcdea0e4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -51,6 +51,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, @@ -121,6 +122,54 @@ _DECODED_JSON: Final = TypeAdapter(object) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: + if custom_llm_provider == "anthropic": + return True + model_lower: Final = model.lower() + if custom_llm_provider == "bedrock": + return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) + if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": + return "claude" in model_lower + return False + + +def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool: + return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model) + + +def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None: + if not isinstance(custom, dict) or "defer_loading" in tool_dict: + return None + defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading") + return defer_loading if isinstance(defer_loading, bool) else None + + +def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object: + if not isinstance(tool, dict): + return tool + + if "custom" not in tool: + return tool + + tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool) + + custom: Final = tool_dict["custom"] + without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"} + defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None + if defer_loading is None: + return without_custom if drop_custom else tool_dict + + unhoisted: Final = { + key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading" + } + kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {} + return {**without_custom, "defer_loading": defer_loading, **kept_custom} + + +def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]: + return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) @@ -552,6 +601,20 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]: + return ( + *( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) + if self.is_thinking_display_updates_used(thinking) + else () + ), + *( + (ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,) + if isinstance(thinking, dict) and "block_binding" in thinking + else () + ), + ) + def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool: for message in messages: if message_field(message, "role") != "system": @@ -1132,7 +1195,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): *, custom_llm_provider: str, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, ) -> list[str]: """ @@ -1169,13 +1232,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_mid_conversation_output_config_used: betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else () - ) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else () ) - return list(set(betas).union(thinking_display_betas, tool_change_betas)) + return list(set(betas).union(self.thinking_betas(thinking), tool_change_betas)) @staticmethod def _make_api_key_auth_header( @@ -1237,7 +1297,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: str | None = None, use_bearer_for_custom_base: bool = False, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, wif_minted: bool = False, ) -> dict: @@ -1298,7 +1358,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.update(user_anthropic_beta_headers) all_betas: Final = betas.union( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (), + self.thinking_betas(thinking), (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (), ) @@ -1385,7 +1445,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): auth_token=auth_token, file_id_used=file_id_used, is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")), + thinking=optional_params.get("thinking"), is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages), web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index a230294ad3f..f8bf7232d9c 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drop_params, ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..91fc77773d1 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -11,6 +11,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountFields, @@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + drop_params=drops_unsupported_params(litellm_params.get("drop_params")), ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 0967c22f93a..e60ffe67c8d 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import split_leading_system_run, ) from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + is_claude_model_target, + merge_anthropic_beta_headers, + tools_without_custom_field, +) from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -372,6 +376,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + drop_params: bool = False, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ Transform request to Anthropic CountTokens format. @@ -391,7 +396,19 @@ class AnthropicCountTokensConfig: **MappingProxyType( { key: value - for key, value in (("system", counted_system), ("tools", tools)) + for key, value in ( + ("system", counted_system), + ( + "tools", + None + if tools is None + else tools_without_custom_field( + tools, + hoist_defer_loading=is_claude_model_target("anthropic", model), + drop_custom=drop_params, + ), + ), + ) if value is not None } ), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..3524c9e73d8 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -203,7 +203,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ToolMessageContentPart, ) -from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper @@ -362,6 +362,9 @@ class AnthropicAdapter: _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) +_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value}) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -817,6 +820,7 @@ class LiteLLMAnthropicMessagesAdapter: "name", "input_schema", "description", + "custom", "cache_control", "strict", "type", @@ -1127,7 +1131,8 @@ class LiteLLMAnthropicMessagesAdapter: """Translate Anthropic thinking to either thinking or reasoning_effort. A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one - speaks that param. Carrying its adaptive effort tier alongside takes two different params, + speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only + ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params, because the two are not interchangeable at the provider mapping below. Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking`` @@ -1158,7 +1163,9 @@ class LiteLLMAnthropicMessagesAdapter: is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + is_claude_target: Final = ( + self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS output_config: Final = anthropic_message_request.get("output_config") if is_claude_target: diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..c086618e829 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -2,10 +2,22 @@ from collections.abc import AsyncIterator, Mapping, Sequence from typing import ClassVar, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.anthropic_beta_gated_fields import ( + log_beta_gated_strips, + strip_beta_gated_native_request, + strip_beta_gated_system_messages, + unstripped_native_request, +) +from litellm.anthropic_beta_headers_manager import ( + get_unsupported_headers, + has_beta_headers_config, + is_beta_header_supported_by_bundled_config, +) +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -19,7 +31,6 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, - ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, AnthropicMessagesRequest, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -29,13 +40,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, + is_claude_model_target, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, + tools_without_custom_field, ) from .mid_conversation_system import ( as_system_content_blocks, @@ -47,6 +61,8 @@ DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" _CALLER_CREDENTIAL_HEADERS: Final = frozenset({"x-api-key", "authorization"}) +_MESSAGE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + def _carries_caller_credential(headers: Mapping[str, str]) -> bool: """Whether the caller sent their own Anthropic credential, in which case this passthrough @@ -81,6 +97,9 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa return response +_MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True @@ -181,6 +200,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return is_claude_model_target(self._resolved_provider, model) + + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools_without_custom_field( + tools, + hoist_defer_loading=self.hoists_wrapped_defer_loading(model), + drop_custom=drops_unsupported_params(drop_params), + ) + @staticmethod def _filter_billing_headers_from_system(system_param): """ @@ -194,7 +223,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -205,7 +234,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: @@ -215,7 +244,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: return system_param - def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: + def _strip_beta_gated_system_turns( + self, messages: Sequence[Mapping[str, object]], model: str, drop_params: object + ) -> Sequence[Mapping[str, object]]: + if not has_beta_headers_config(self._resolved_provider): + return messages + gated: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) + return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else messages + + def _normalize_system_role_messages( + self, anthropic_messages_request: dict, model: str, drop_params: object + ) -> None: """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, Vertex, and Azure Foundry all enforce identically. @@ -263,7 +303,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self.custom_llm_provider, key="supports_mid_conversation_system", ) - else list(convert_mid_conversation_system_turns(messages[leading_count:])) + else list( + convert_mid_conversation_system_turns( + self._strip_beta_gated_system_turns( + messages[leading_count:], # pyright: ignore[reportUnknownArgumentType] # messages comes from the bare request dict + model=model, + drop_params=drop_params, + ) + ) + ) ) if hoisted or remaining != messages: anthropic_messages_request["messages"] = remaining @@ -322,7 +370,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(headers, optional_params, messages), api_base + return self.finalize_messages_headers(headers, optional_params, messages), api_base async def avalidate_anthropic_messages_environment( self, @@ -362,7 +410,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base + return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]: if auth_header is None: @@ -396,7 +444,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return config_allows_workload_identity(self) - def _finalize_messages_headers( + def finalize_messages_headers( self, headers: dict, # mutable-ok: out-param optional_params: dict, # mutable-ok: out-param @@ -605,6 +653,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if thinking_enabled or effort_enabled: optional_params.pop("temperature", None) + def strips_beta_gated_fields_from_final_headers(self) -> bool: + """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke), + which strip beta-gated fields themselves once that list exists, and on providers with no row in + ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail.""" + return has_beta_headers_config(self._resolved_provider) + def transform_anthropic_messages_request( self, model: str, @@ -700,20 +754,47 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. - _tools: Final = anthropic_messages_optional_request_params.get("tools") or [] + tools: Final = anthropic_messages_optional_request_params.get("tools") + _tools: Final = tools or [] _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools) if not _has_advisor: messages = strip_advisor_blocks_from_messages(messages) + gated: Final = ( + strip_beta_gated_native_request( + headers, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + messages, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare list + drop_params=getattr(litellm_params, "drop_params", None), + ) + if self.strips_beta_gated_fields_from_final_headers() + else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + ) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( - drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + drop_non_string_tool_result_texts( + _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages, + getattr(litellm_params, "drop_params", None), + ) ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, ) - return dict(anthropic_messages_request) + request: Final = ( + {**dict(anthropic_messages_request), "thinking": gated.thinking} + if gated.thinking_stripped + else dict(anthropic_messages_request) # pyright: ignore[reportUnknownVariableType] # built from the base signature's bare dict + ) + if not isinstance(tools, list): + return request + return { + **request, + "tools": self.tools_for_upstream( + OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params + ), + } def transform_anthropic_messages_response( self, @@ -844,17 +925,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if AnthropicModelInfo().is_tool_search_used(tools): beta_values.add(get_tool_search_beta_header(custom_llm_provider)) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) - if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking")) - else () - ) + thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking")) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages) else () ) - all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas) + all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas) dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value forwarded_beta_values: Final = ( # pyright: ignore[reportUnknownVariableType] # derived from the bare beta set all_beta_values - {dangerous_tool_use} diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 99a07a1e9bf..9b22170c9db 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import ( supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} +def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]: + if "prompt_cache_breakpoint" not in block: + return cache_control_as_prompt_cache_breakpoint(block) + wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True) + return cache_control_as_prompt_cache_breakpoint( + with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker) + ) + + def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None: """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" + drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): - return cache_control_as_prompt_cache_breakpoint + return ( + _cache_control_as_wire_safe_prompt_cache_breakpoint + if drops_params + else cache_control_as_prompt_cache_breakpoint + ) if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + if drops_params: return _without_prompt_cache_breakpoint return None diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 17c178d169e..27e003ce8b6 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -661,12 +661,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + config: Final = AnthropicMessagesConfig() + body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_optional_request_params=filtered, litellm_params=GenericLiteLLMParams(), - headers={}, + headers=config.finalize_messages_headers({}, dict(filtered), list(messages)), # pyright: ignore[reportUnknownMemberType] # the shared helper takes and returns bare dicts ) return MappingProxyType(_JSON_OBJECT.validate_python(body)) except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 5a181faad11..a1ba8de6d7f 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -33,6 +33,7 @@ from litellm.types.utils import ( from litellm.utils import ( CustomStreamWrapper, convert_to_model_response_object, + get_base_model_from_metadata, ) from ...types.llms.openai import HttpxBinaryResponseContent @@ -145,6 +146,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): azure_client: AzureOpenAI | OpenAI, data: dict, timeout: float | httpx.Timeout, + logging_obj: LiteLLMLoggingObj, ): """ Helper to: @@ -160,7 +162,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=False) + return build_output_token_limit_response( + e=e, + data=data, + is_async=False, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: raise e @@ -192,7 +199,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=True) + return build_output_token_limit_response( + e=e, + data=data, + is_async=True, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: raise e @@ -350,7 +362,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): ) headers, response = self.make_sync_azure_openai_chat_completion_request( - azure_client=azure_client, data=data, timeout=timeout + azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj ) if not isinstance(response, BaseModel): raise AzureOpenAIError( @@ -564,7 +576,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): }, ) headers, response = self.make_sync_azure_openai_chat_completion_request( - azure_client=azure_client, data=data, timeout=timeout + azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj ) logging_obj.model_call_details["response_headers"] = headers streamwrapper: Final = CustomStreamWrapper( diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index eb6d103efb9..09777da8669 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -335,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data @@ -343,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - GET /v1/responses/{response_id} """ get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) - data: Final[dict] = {} + data: Final[dict[str, str]] = {} verbose_logger.debug("get response url=%s", get_url) return get_url, data diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 0f8afaae681..4b3f263201a 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -13,6 +13,7 @@ from pydantic import JsonValue import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, @@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")), ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/azure_ai/anthropic/handler.py b/litellm/llms/azure_ai/anthropic/handler.py index ca4dad5ee3e..1a2003e6262 100644 --- a/litellm/llms/azure_ai/anthropic/handler.py +++ b/litellm/llms/azure_ai/anthropic/handler.py @@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import httpx +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from litellm.llms.anthropic.chat.transformation import JsonModeRequest @@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): litellm_params=litellm_params, ) - data: Final = config.transform_request( + request_body: Final[Mapping[str, object]] = config.transform_request( model=model, messages=messages, optional_params=optional_params, litellm_params=litellm_params, headers=headers, ) + data: Final = dict( + strip_beta_gated_request_body( + request_body, + headers, # pyright: ignore[reportUnknownArgumentType] # validate_environment returns a bare dict + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 6b2da96de23..1f9ded827b6 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -145,6 +145,8 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) return anthropic_messages_request diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index c75213c3e2c..6516827359d 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -107,9 +107,11 @@ class AzureAnthropicConfig(AnthropicConfig): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) # Merge headers - Azure auth (api-key or Authorization) takes precedence - headers = {**anthropic_headers, **headers} + merged_beta_header: Final = {k: v for k, v in anthropic_headers.items() if k == "anthropic-beta"} + headers = {**anthropic_headers, **headers, **merged_beta_header} # Ensure anthropic-version header is set if "anthropic-version" not in headers: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..8bb6072d0a9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -106,7 +106,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 35b7642d7de..2e5514b3975 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -5,8 +5,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm._lazy_imports import get_default_encoding from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, @@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import ( relayed_body, strip_leading_model_segment, ) +from litellm.llms.cohere.embed.transformation import ( + CohereEmbeddingRowShape, + CohereImageTokenBilling, + cohere_embedding_response, +) from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse from litellm.types.llms.openai import AllMessageValues from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response @@ -35,6 +41,7 @@ if TYPE_CHECKING: EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) +ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True)) def api_version_from(litellm_params: Mapping[str, object]) -> str | None: @@ -85,9 +92,29 @@ def relay_query_params( return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) +def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]: + try: + return ECHOED_TEXTS.validate_python(body.get("texts")) + except ValidationError: + return () + + +def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse: + return cohere_embedding_response( + body, + None, + get_default_encoding(), + echoed_texts(body), + CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS, + CohereEmbeddingRowShape.TYPED, + ) + + FOUNDRY_RELAY_SHAPES: Final = ( RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), + RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response), + RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response), ) diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index da734cf8189..b19915648ab 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -232,7 +232,7 @@ class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: mapped: Final = super().map_openai_params_for_deployment( response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 01baeed9bef..792b14090b8 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -69,6 +69,9 @@ class BaseResponsesAPIConfig(ABC): def supports_encrypted_agent_messages(self) -> bool: return False + def supports_streamed_retrieval(self) -> bool: + return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI) + def sign_request( self, headers: dict, @@ -209,7 +212,7 @@ class BaseResponsesAPIConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: pass @abstractmethod @@ -268,6 +271,9 @@ class BaseResponsesAPIConfig(ABC): """Returns True if litellm should fake a stream for the given model and stream value""" return False + def may_answer_stream_with_json(self) -> bool: + return False + def supports_native_websocket(self) -> bool: """ Returns True if the provider has a native WebSocket endpoint for Responses API. diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..443e722168d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, @@ -117,6 +119,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1192,11 +1195,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params @@ -1828,7 +1827,16 @@ class AmazonConverseConfig(BaseConfig): filtered_tools.append(tool) base_model: Final = BedrockModelInfo.get_base_model(model) - client_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + header_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + client_beta_list: Final = header_beta_list or filter_and_transform_beta_headers( + sorted( + active_beta_values( + {}, + additional_request_params.get("anthropic_beta"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + ), + provider="bedrock_converse", + ) eager_beta: Final = ( (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,) if base_model.startswith("anthropic") @@ -1836,7 +1844,15 @@ class AmazonConverseConfig(BaseConfig): and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list else () ) - anthropic_beta_list: Final = [*client_beta_list, *eager_beta] + thinking_betas: Final = AnthropicModelInfo().thinking_betas( + additional_request_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + missing_thinking_betas: Final = ( + tuple(beta for beta in thinking_betas if beta not in client_beta_list) + if base_model.startswith("anthropic") + else () + ) + anthropic_beta_list: Final = [*client_beta_list, *eager_beta, *missing_thinking_betas] # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): @@ -1906,7 +1922,8 @@ class AmazonConverseConfig(BaseConfig): else: computer_use_header = "computer-use-2024-10-22" - anthropic_beta_list.append(computer_use_header) + if computer_use_header not in anthropic_beta_list: + anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format transformed_computer_tools: Final = self._transform_computer_use_tools(computer_use_tools) additional_request_params["tools"] = transformed_computer_tools @@ -2031,6 +2048,16 @@ class AmazonConverseConfig(BaseConfig): bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( original_tools, model, headers, additional_request_params, litellm_params, deployment_model ) + requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict + body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict + gated_thinking: Final = strip_beta_gated_thinking( + requested_thinking, # pyright: ignore[reportUnknownArgumentType] # same bare dict + active_beta_values({}, body_betas) | frozenset(anthropic_beta_list), # pyright: ignore[reportUnknownArgumentType] # _process_tools_and_beta hands back a bare list + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse") + additional_request_params["thinking"] = gated_thinking.thinking # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index eb6ed5c75e3..e32df0bbdbb 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import TypeAdapter +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if beta_list: _anthropic_request["anthropic_beta"] = beta_list - return _anthropic_request + return dict( + strip_beta_gated_request_body( + _anthropic_request, # pyright: ignore[reportUnknownArgumentType] # built by the base signature's bare dict transform + {}, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the base signature's bare dict + model=model, + provider="bedrock_invoke", + ) + ) @property def uses_async_transform_request(self) -> bool: @@ -294,7 +303,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) - is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking) client_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) user_beta_set: Final = ( @@ -312,7 +320,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=is_thinking_display_updates_used, + thinking=thinking, ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 0194f1c7e45..a5ecc37d394 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): ), code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")), container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params), + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index fde995c7907..4175479f068 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -15,7 +15,11 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ( + ReadOnly, + TypedDict, + TypeIs, # noqa: TID251 # narrows the caller's tools without copying them +) if TYPE_CHECKING: from botocore.eventstream import EventStreamMessage @@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys( request_body["output_config"] = {"format": preserved_format} # rebind-ok: out-param +_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: JSON tool objects # mutable-ok: fallbacks reuse tools edited in place + return isinstance(value, dict) + + +def _normalize_custom_field_on_tool(tool: object) -> None: + if not _is_str_keyed_dict(tool): + return + custom: Final = tool.pop("custom", None) + if not _is_str_keyed_dict(custom) or "defer_loading" in tool: + return + deferred: Final = custom.get("defer_loading") + if isinstance(deferred, bool): + tool["defer_loading"] = deferred # rebind-ok: a fallback reuses the caller's tool, as on main + + def normalize_custom_field_on_tools(request_body: dict) -> None: """ Drop the ``custom`` field from each tool, first hoisting a boolean @@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None: tools: Final = request_body.get("tools") if not tools or not isinstance(tools, list): return - for tool in tools: - if not isinstance(tool, dict): - continue - custom: dict[str, object] | None = tool.pop("custom", None) - if not isinstance(custom, dict) or "defer_loading" in tool: - continue - deferred: object = custom.get("defer_loading") - if isinstance(deferred, bool): - tool["defer_loading"] = deferred + for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools): + _normalize_custom_field_on_tool(tool) _TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -1314,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 34e622b5e8d..5cd26235ba2 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url @@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None ) endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name) + tools: Final = request.get("tools") body: Final = self._anthropic_config.transform_request_to_count_tokens( model=resolved_model, messages=request["messages"], - tools=request.get("tools"), + tools=None + if tools is None + else _TOOLS.validate_python( + tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True) + ), system=request.get("system"), optional_params=_COUNT_OPTIONS.validate_python( {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES} diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index bf9c119f6f2..df5a3827c1f 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -12,6 +12,7 @@ from typing import Final, Literal from pydantic import JsonValue from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import tools_without_custom_field from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import get_bedrock_base_model @@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM): # For InvokeModel, we need to provide the raw body that would be sent to the model # Remove the 'model' field from the body as it's not part of the model input - body_data: Final = {k: v for k, v in request_data.items() if k != "model"} + tools: Final = request_data.get("tools") + body_data: Final = { + **{k: v for k, v in request_data.items() if k != "model"}, + **( + {"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)} + if isinstance(tools, list) + else {} + ), + } if "messages" in body_data: # Bedrock validates the body against the model's InvokeModel schema; diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 4d27d94664f..2f0ed4d0e82 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,9 +9,11 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx +from pydantic import TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL +from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( @@ -36,7 +38,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError +from ..common_utils import BedrockError, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -64,12 +66,21 @@ def _sign_get_request( return request.prepare() +_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def _without_region_path(model: str) -> str: + region, model_id = split_bedrock_region_path(model) + return model if region is None else model_id + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, + region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -77,15 +88,18 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, + region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, + region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) + explicit_region_name: Final = optional_params.pop("aws_region_name", None) + aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name ### SET REGION NAME ### if aws_region_name is None: @@ -184,14 +198,17 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, is_async_invoke: bool | None = False, batch_data: list[dict] | None = None, + aws_region_name: str | None = None, ) -> EmbeddingResponse | None: """ Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None + bedrock_model: Final = _without_region_path(model) + is_async_job: Final = bool(is_async_invoke) and len(response_list) == 1 and "invocationArn" in response_list[0] # Handle async invoke responses (single response with invocationArn) - if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: + if is_async_job: if provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response( response=response_list[0], model=model @@ -227,15 +244,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -251,6 +268,8 @@ class BedrockEmbedding(BaseAWSLLM): ########################################################## if returned_response is None: raise Exception(f"Unable to map model response to known provider format. model={model}") + if aws_region_name is not None and not is_async_job: + set_hidden_param(returned_response, "region_name", aws_region_name) return returned_response def _single_func_embeddings( @@ -319,6 +338,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) async def _async_single_func_embeddings( @@ -390,6 +410,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) def embeddings( @@ -409,26 +430,29 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: + model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None + has_async_invoke: Final = "async_invoke/" in model + model_without_async_route: Final = model.replace("async_invoke/", "", 1) + path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None + bedrock_model: Final = _without_region_path(model_without_async_route) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key) + optional_params, + bearer_token=bedrock_bearer_token(api_key), + region_from_model=self._get_aws_region_from_model_arn(model_id_param) or path_region, ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed + unencoded_model_id: Final = model_id_param or bedrock_model modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model, + model=bedrock_model, model_id=unencoded_model_id, ) - # Check async invoke needs to be used - has_async_invoke: Final = "async_invoke/" in model - if has_async_invoke: - model = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(model) + provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {model}. " + f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -441,9 +465,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=model, input=input, inference_params=inference_params + model=bedrock_model, input=input, inference_params=inference_params ) - elif provider == "amazon" and model in [ + elif provider == "amazon" and bedrock_model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -451,28 +475,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - model, + bedrock_model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -485,7 +509,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -527,7 +551,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -541,7 +565,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -570,8 +594,9 @@ class BedrockEmbedding(BaseAWSLLM): ## ROUTING ## # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} + set_hidden_param(model_response, "region_name", aws_region_name) return cohere_embedding( - model=model, + model=model_without_async_route, input=input, model_response=model_response, logging_obj=logging_obj, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index e1e15861a90..e5d6a168d57 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -7,6 +7,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, @@ -102,6 +103,9 @@ class AmazonAnthropicClaudeMessagesConfig( def should_strip_billing_metadata(self) -> bool: return True + def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]: + return tools + def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool: return True @@ -546,6 +550,9 @@ class AmazonAnthropicClaudeMessagesConfig( "edits": retained_edits, } + def strips_beta_gated_fields_from_final_headers(self) -> bool: + return False + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, @@ -581,9 +588,7 @@ class AmazonAnthropicClaudeMessagesConfig( ), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used( - anthropic_messages_request.get("thinking") - ), + thinking=anthropic_messages_request.get("thinking"), is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used( outgoing_messages_typed ), @@ -754,7 +759,9 @@ class AmazonAnthropicClaudeMessagesConfig( ), "display": "updates", } - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### @@ -822,6 +829,20 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict + gated: Final = strip_beta_gated_native_request( + {}, + final_request, + final_request.get("messages"), + body_betas=filtered_betas, + drop_params=getattr(litellm_params, "drop_params", None), + ) + log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke") + if gated.thinking_stripped: + anthropic_messages_request["thinking"] = gated.thinking + if gated.messages_stripped: + anthropic_messages_request["messages"] = list(gated.messages) + outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request) if outbound_tools is not None: anthropic_messages_request["tools"] = outbound_tools diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/bigmodel/__init__.py b/litellm/llms/bigmodel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/__init__.py b/litellm/llms/bigmodel/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py new file mode 100644 index 00000000000..12d5e912c15 --- /dev/null +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.zai.chat.transformation import ZAIChatConfig + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking + +_SUPPORTED_PARAMS: Final = ( + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "temperature", + "top_p", + "stop", + "tools", + "tool_choice", + "response_format", + "thinking", + "reasoning_effort", +) + + +class BigModelChatConfig(ZAIChatConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited interface requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + mapped: Final = { + **optional_params, + **{ + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in _SUPPORTED_PARAMS and key != "thinking" + }, + } + if "thinking" not in non_default_params: + return mapped + thinking: Final = non_default_params["thinking"] + if is_unsupported_disabled_thinking(model, thinking): + return { + **mapped, + "reasoning_effort": mapped.get("reasoning_effort", "low"), + "extra_body": {"thinking": {"type": "enabled"}}, + } + return {**mapped, "extra_body": {"thinking": thinking}} diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py new file mode 100644 index 00000000000..0c9b6da5c8f --- /dev/null +++ b/litellm/llms/bigmodel/common_utils.py @@ -0,0 +1,92 @@ +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit, urlunsplit + +import httpx +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str + +BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""") +_CODING_CHAT_PATH: Final = "/api/coding/paas/v4" +_API_PATHS: Final = ( + "/api/coding/paas/v4/chat/completions", + _CODING_CHAT_PATH, + "/api/paas/v4/chat/completions", + "/api/anthropic/v1/messages", + "/api/anthropic/v1", + "/api/v1/responses", + "/api/paas/v4", + "/api/anthropic", + "/api/v1", + "/api", +) +_ENDPOINT_PATHS: Final = MappingProxyType( + { + "chat": "/api/paas/v4", + "responses": "/api/v1/responses", + "messages": "/api/anthropic/v1/messages", + } +) + + +def get_bigmodel_api_key(api_key: str | None) -> str | None: + return api_key or get_secret_str("BIGMODEL_API_KEY") + + +def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str: + resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE + parsed: Final = urlsplit(resolved_base) + path: Final = parsed.path.rstrip("/") + suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "") + endpoint_path: Final = ( + _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint] + ) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment) + ) + + +def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]: + try: + return _JSON_OBJECT.validate_json(response.content) + except ValidationError: + raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers) + + +def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool: + return thinking == {"type": "disabled"} and litellm.utils.supports_factory( + model=model, custom_llm_provider="bigmodel", key="thinking_always_on" + ) + + +def bigmodel_business_error( + code: JsonValue, message: str, model: str, response: httpx.Response | None +) -> ContextWindowExceededError | ContentPolicyViolationError | None: + match code: + case 1261 | "1261": + return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response) + case 1301 | "1301": + return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response) + case _: + return None + + +def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None: + found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str) + response: Final = getattr(original_exception, "response", None) + error: Final = bigmodel_business_error( + found.group(1) if found else None, + error_str, + str(model), + response if isinstance(response, httpx.Response) else None, + ) + if error is not None: + raise error diff --git a/litellm/llms/bigmodel/messages/__init__.py b/litellm/llms/bigmodel/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py new file mode 100644 index 00000000000..82ce59631f1 --- /dev/null +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -0,0 +1,206 @@ +import json +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Final + +import httpx +from openai._streaming import ServerSentEvent, SSEDecoder +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai_like.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + OpenAILikeAnthropicMessagesConfig, +) +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from litellm.types.router import GenericLiteLLMParams + +from ..common_utils import ( + bigmodel_json_body, + get_bigmodel_api_key, + get_bigmodel_url, + is_unsupported_disabled_thinking, +) + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object]) + + +def _with_lowest_effort_thinking( + params: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: the parent transform takes a request dict + requested_output_config: Final = params.get("output_config") + output_config: Final = { + **({} if "reasoning_effort" in params else {"effort": "low"}), + **( + _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {} + ), + } + return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})} + + +def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response + usage: Final = body.get("usage") + if not isinstance(usage, dict): + return dict(body) + server_tools: Final = usage.get("server_tool_use") + if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools: + return dict(body) + normalized: Final[JsonValue] = { + **body, + "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}}, + } + return normalized + + +class _BigModelMessagesStream(httpx.AsyncByteStream): + def __init__(self, response: httpx.Response) -> None: + self._response = response + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()): + yield _normalize_event(event) + + async def aclose(self) -> None: + await self._response.aclose() + + +def _json_object(data: str) -> Mapping[str, JsonValue] | None: + try: + return _JSON_OBJECT.validate_json(data) + except ValidationError: + return None + + +def _normalized_data(data: str) -> str: + body: Final = _json_object(data) + if body is None: + return data + message: Final = body.get("message") + normalized: Final[JsonValue] = ( + {**body, "message": _normalize_message(message)} + if body.get("type") == "message_start" and isinstance(message, dict) + else _normalize_message(body) + ) + return data if normalized == body else json.dumps(normalized, ensure_ascii=False) + + +def _normalize_event(event: ServerSentEvent) -> bytes: + metadata: Final = "".join( + f"{name}: {value}\n" + for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) + if value is not None + ) + data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n")) + return f"{metadata}{data_lines}\n".encode() + + +class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def is_native_messages_route(self) -> bool: + return True + + def should_strip_billing_metadata(self) -> bool: + return True + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict[str, object]], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: matches dict-typed base signature + ) -> dict[str, object]: # mutable-ok: matches dict-typed base signature + request_params: Final = ( + _with_lowest_effort_thinking(anthropic_messages_optional_request_params) + if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking")) + else anthropic_messages_optional_request_params + ) + return super().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the parent transform returns a bare dict + model, messages, request_params, litellm_params, headers + ) + + def transform_anthropic_messages_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> AnthropicMessagesResponse: + normalized: Final = httpx.Response( + raw_response.status_code, + json=_normalize_message(bigmodel_json_body(raw_response)), + request=raw_response.request, + ) + return super().transform_anthropic_messages_response(model, normalized, logging_obj) + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict[str, object], # mutable-ok: inherited native stream API takes a request dict + litellm_logging_obj: Logging, + ) -> AsyncIterator[bytes]: + normalized: Final = httpx.Response( + httpx_response.status_code, + headers={ + key: value + for key, value in httpx_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + stream=_BigModelMessagesStream(httpx_response), + request=httpx_response.request, + ) + return super().get_async_streaming_response_iterator( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # native parent yields bytes but its iterator annotation omits the element type + model, normalized, request_body, litellm_logging_obj + ) + + @staticmethod + def _translate_reasoning_effort_to_anthropic( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _translate_adaptive_effort_for_non_adaptive_model( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _drop_incompatible_temperature_for_thinking( + model: str, optional_params: Mapping[str, object], custom_llm_provider: str + ) -> None: + return None + + def validate_anthropic_messages_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(api_key) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")}, + "x-api-key": resolved_key, + "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, + "content-type": "application/json", + }, api_base + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return get_bigmodel_url(api_base, "messages") diff --git a/litellm/llms/bigmodel/responses/__init__.py b/litellm/llms/bigmodel/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py new file mode 100644 index 00000000000..80c53682d16 --- /dev/null +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -0,0 +1,263 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError +from typing_extensions import Never + +from litellm.exceptions import AuthenticationError, UnsupportedParamsError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SUPPORTED_PARAMS: Final = ( + "input", + "model", + "instructions", + "stream", + "temperature", + "top_p", + "max_output_tokens", + "tools", + "tool_choice", + "parallel_tool_calls", + "include", + "reasoning", + "text", + "prompt_cache_key", + "previous_response_id", + "store", + "extra_headers", + "extra_query", + "extra_body", + "timeout", +) + + +_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType( + { + **dict.fromkeys((1000, 1001, 1003, 1005), 401), + **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400), + 1220: 403, + **dict.fromkeys( + (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321), + 429, + ), + } +) + + +class _RequestedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + tools: JsonValue = None + tool_choice: JsonValue = None + parallel_tool_calls: JsonValue = None + + +def _requested_tools(logging_obj: Logging) -> _RequestedTools: + try: + return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {}) + except ValidationError: + return _RequestedTools() + + +def _envelope_status_code(code: JsonValue) -> int: + if not isinstance(code, int) or isinstance(code, bool): + return 500 + if 400 <= code <= 599: + return code + return _STATUS_BY_BUSINESS_CODE.get(code, 500) + + +def _normalize_output_item(item: JsonValue) -> JsonValue: + if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): + return item + content: Final = item.get("content") + normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item + if not isinstance(content, dict): + return normalized + return {**normalized, "content": [content]} + + +def _normalize_response( + body: Mapping[str, JsonValue], + requested: _RequestedTools, +) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary + usage: Final = body.get("usage") + output: Final = body.get("output") + tools: Final = body.get("tools") + tool_choice: Final = body.get("tool_choice") + parallel_tool_calls: Final = body.get("parallel_tool_calls") + input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None + output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None + normalized_usage: Final = ( + { + "total_tokens": input_tokens + output_tokens, + **usage, + } + if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int) + else usage + ) + normalized: Final[JsonValue] = { + **body, + "parallel_tool_calls": ( + requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls + ), + "tools": (requested.tools or []) if tools is None else tools, + "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, + **({"usage": normalized_usage} if "usage" in body else {}), + **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), + } + return normalized + + +def _normalize_stream_part(part: JsonValue) -> JsonValue: + if not isinstance(part, dict): + return part + if part.get("type") == "output_text": + return { + "annotations": [], + "logprobs": None, + **part, + } + if part.get("type") == "reasoning_text": + return {"reasoning": part.get("text", ""), **part} + return part + + +class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BIGMODEL + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return False + + def may_answer_stream_with_json(self) -> bool: + return True + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS} + + def validate_environment( + self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict[str, str]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **headers, + "Authorization": f"Bearer {resolved_key}", + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + return get_bigmodel_url(api_base, "responses") + + def transform_response_api_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + body: Final = bigmodel_json_body(raw_response) + if body.get("success") is False: + message: Final = str(body.get("msg") or raw_response.text) + raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException( + message=message, + status_code=_envelope_status_code(body.get("code")), + headers=raw_response.headers, + ) + normalized: Final = _normalize_response(body, _requested_tools(logging_obj)) + normalized_response: Final = httpx.Response( + status_code=raw_response.status_code, + headers={ + key: value + for key, value in raw_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + json=normalized, + request=raw_response.request, + ) + return super().transform_response_api_response(model, normalized_response, logging_obj) + + def transform_get_response_api_response( + self, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + return self.transform_response_api_response("", raw_response, logging_obj) + + def transform_streaming_response( + self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging + ) -> ResponsesAPIStreamingResponse: + chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk) + response: Final = chunk.get("response") + item: Final = chunk.get("item") + normalized: Final[JsonValue] = { + **chunk, + **( + {"response": _normalize_response(response, _requested_tools(logging_obj))} + if isinstance(response, dict) + else {} + ), + **({"item": _normalize_output_item(item)} if "item" in chunk else {}), + } + event_model: Final = self.get_event_model_class(str(normalized.get("type"))) + match chunk.get("type"): + case "response.output_text.done": + return event_model.model_validate({"logprobs": [], **normalized}) + case "response.content_part.done": + return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))}) + case "error": + error: Final = chunk.get("error") + error_fields: Final = error if isinstance(error, dict) else chunk + normalized_error: Final[JsonValue] = { + **normalized, + "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"}, + } + return event_model.model_validate(normalized_error) + case _: + return event_model.model_validate(normalized) + + def transform_cancel_response_api_request( + self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str] + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400) + + def transform_compact_response_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: Mapping[str, object], + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400) diff --git a/litellm/llms/cohere/embed/transformation.py b/litellm/llms/cohere/embed/transformation.py index 590164ffa0f..1477e45997b 100644 --- a/litellm/llms/cohere/embed/transformation.py +++ b/litellm/llms/cohere/embed/transformation.py @@ -10,11 +10,13 @@ Convers Docs - https://docs.cohere.com/v2/reference/embed """ -from collections.abc import Iterable, Mapping, Sized -from typing import Final, Protocol, cast +from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized +from enum import Enum +from itertools import chain, starmap +from typing import Final, Protocol, TypeAlias, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter import litellm from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE @@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True)) _DECODED_JSON: Final = TypeAdapter(object) class _SupportsEncode(Protocol): """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" - def encode(self, text: str, /) -> Sized: ... + def encode_ordinary(self, text: str, /) -> Sized: ... + + +class CohereImageTokenBilling(Enum): + SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens" + FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens" + + +class CohereEmbeddingRowShape(Enum): + OPENAI = "openai" + TYPED = "typed" + + +class _CohereBilledUnits(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + input_tokens: int | None = None + image_tokens: int | None = None + images: int | None = None + + +class _CohereEmbedMeta(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits) + + +_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object] + + +class _CohereEmbedBody(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + embeddings: _CohereEmbeddings + meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta) + + +def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors)) + + +def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({**row, "type": embedding_type} for row in _embedding_rows(vectors)) + + +def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]: + if not isinstance(embeddings, Mapping): + return _embedding_rows(embeddings) + match row_shape: + case CohereEmbeddingRowShape.OPENAI: + return chain.from_iterable(map(_embedding_rows, embeddings.values())) + case CohereEmbeddingRowShape.TYPED: + return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items())) + + +def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=prompt_tokens_details, + ) + + +def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage: + match image_billing: + case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS: + image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0) + return _prompt_usage( + (billed_units.input_tokens or 0) + image_units, + PromptTokensDetailsWrapper( + text_tokens=billed_units.input_tokens, + image_tokens=image_units or None, + image_count=billed_units.images, + ), + ) + case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS: + return _prompt_usage(billed_units.input_tokens or 0, None) + + +def _cohere_embed_usage( + billed_units: _CohereBilledUnits, + input_texts: Iterable[str], + encoding: _SupportsEncode, + image_billing: CohereImageTokenBilling, +) -> Usage: + if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None: + return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None) + return _billed_usage(billed_units, image_billing) + + +def cohere_embedding_response( + body: Mapping[str, object], + model: str | None, + encoding: _SupportsEncode, + input_texts: Iterable[str], + image_billing: CohereImageTokenBilling, + row_shape: CohereEmbeddingRowShape, +) -> EmbeddingResponse: + cohere_body: Final = _CohereEmbedBody.model_validate(body) + return EmbeddingResponse( + model=model, + data=list(_response_rows(cohere_body.embeddings, row_shape)), + usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing), + ) class CohereEmbeddingConfig(BaseEmbeddingConfig): @@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): ), ) - def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: - input_tokens = 0 - - text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") - - image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images") - - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if image_tokens is None and text_tokens is None: - for text in input: - input_tokens += len(encoding.encode(text)) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - image_tokens=image_tokens, - text_tokens=text_tokens, - ) - if image_tokens: - input_tokens += image_tokens - if text_tokens: - input_tokens += text_tokens - - return Usage( - prompt_tokens=input_tokens, - completion_tokens=0, - total_tokens=input_tokens, - prompt_tokens_details=prompt_tokens_details, - ) - - def _transform_response( + def transform_native_embed_response( self, response: httpx.Response, api_key: str | None, logging_obj: LiteLLMLoggingObj, - data: dict | CohereEmbeddingRequest, - model_response: EmbeddingResponse, + data: Mapping[str, object] | CohereEmbeddingRequest, model: str, encoding: _SupportsEncode, - input: list, + input: Sequence[str], + row_shape: CohereEmbeddingRowShape, ) -> EmbeddingResponse: response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING @@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): additional_args={"complete_input_dict": data}, original_response=response_json, ) - """ - response - { - 'object': "list", - 'data': [ - - ] - 'model', - 'usage' - } - """ - payload: Final = _JSON_OBJECT.validate_python(response_json) - embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"]) - output_data: Final = [] - for k, embedding_list in embeddings.items(): - for idx, embedding in enumerate(embedding_list): - output_data.append({"object": "embedding", "index": idx, "embedding": embedding}) - model_response.object = "list" - model_response.data = output_data - model_response.model = model - input_tokens = 0 - for text in input: - input_tokens += len(encoding.encode(text)) - - setattr( - model_response, - "usage", - self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))), + return cohere_embedding_response( + _JSON_OBJECT.validate_python(response_json), + model, + encoding, + input, + CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS, + row_shape, ) - return model_response - def transform_embedding_response( self, model: str, @@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, litellm_params: dict, ) -> EmbeddingResponse: - return self._transform_response( + return self.transform_native_embed_response( response=raw_response, api_key=api_key, logging_obj=logging_obj, data=request_data, - model_response=model_response, model=model, encoding=litellm.encoding, input=logging_obj.model_call_details["input"], + row_shape=CohereEmbeddingRowShape.OPENAI, ) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..54adfb5c692 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -764,18 +764,23 @@ class AsyncHTTPHandler: follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, max_response_bytes: int | None = None, + stream: bool = False, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT + query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url)) + request_headers: Final = httpx.Headers(headers) - params = params or {} - params.update(HTTPHandler.extract_query_params(url)) + if stream: + return await self._send_streaming_get( + url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout + ) if max_response_bytes is not None: return await self._get_with_response_limit( url, - params=httpx.QueryParams(params), - headers=httpx.Headers(headers), + params=query, + headers=request_headers, max_bytes=max_response_bytes, follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects, timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout), @@ -783,13 +788,41 @@ class AsyncHTTPHandler: response: Final = await self.client.get( url, - params=params, - headers=headers, + params=query, + headers=request_headers, follow_redirects=_follow_redirects, timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response + async def _send_streaming_get( + self, + url: str, + *, + params: httpx.QueryParams, + headers: httpx.Headers, + follow_redirects: bool | None, + timeout: float | httpx.Timeout | None, + ) -> httpx.Response: + start_time: Final = time.time() + request: Final = self.client.build_request( + "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout + ) + try: + response: Final = await self.client.send( + request, + stream=True, + follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, + ) + _anchor_handler_to(response, self) + response.raise_for_status() + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) + except httpx.HTTPStatusError as e: + await _raise_masked_async_error(e, True) + else: + return response + async def _get_with_response_limit( self, url: str, @@ -1468,15 +1501,20 @@ class HTTPHandler: headers: Mapping[str, Any] | None = None, follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, + stream: bool = False, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT - params = params or {} - params.update(self.extract_query_params(url)) + query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url)) + + if stream: + return self._send_streaming_get( + url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout + ) response: Final = self.client.get( url, - params=params, + params=query, headers=headers, follow_redirects=_follow_redirects, timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, @@ -1484,6 +1522,34 @@ class HTTPHandler: return response + def _send_streaming_get( + self, + url: str, + *, + params: httpx.QueryParams, + headers: Mapping[str, str] | None, + follow_redirects: bool | None, + timeout: float | httpx.Timeout | None, + ) -> httpx.Response: + start_time: Final = time.time() + request: Final = self.client.build_request( + "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout + ) + try: + response: Final = self.client.send( + request, + stream=True, + follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, + ) + _anchor_handler_to(response, self) + response.raise_for_status() + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) + except httpx.HTTPStatusError as e: + _raise_masked_sync_error(e, True) + else: + return response + @staticmethod def extract_query_params(url: str) -> dict[str, str]: """ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index e7da6586b97..4c0f84a003d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -367,12 +371,37 @@ class _SignedBodyKwargs(TypedDict, total=False): json: ReadOnly[dict[str, object]] +def _is_sized_json_body(response: httpx.Response) -> bool: + headers: Final = httpx.Headers(response.headers) + media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type")) + return "content-length" in headers and "application/json" in media_types + + +def _is_json_document(content: bytes) -> bool: + try: + return isinstance(json.loads(content), dict) + except ValueError: + return False + + def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs: if signed_body is not None: return {"data": signed_body} return {"json": data} +def _answered_json(response: httpx.Response) -> bool: + return "application/json" in response.headers.get("content-type", "").lower() + + +def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]: + if not stream: + return {} + if starting_after is None: + return {"stream": "true"} + return {"stream": "true", "starting_after": starting_after} + + def _google_genai_streaming_hidden_params( *, api_base: str, @@ -2552,6 +2581,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **litellm_params.per_second_pricing(), **vertex_location_params, **anthropic_messages_optional_request_params, }, @@ -3025,6 +3055,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = sync_httpx_client.post( @@ -3036,7 +3077,13 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: if fake_stream is True: - return MockResponsesAPIStreamingIterator( + return replay_as_stream(response=response) + if not ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(response.read()) + ): + return SyncResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3046,17 +3093,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return SyncResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) response.read() else: response = sync_httpx_client.post( @@ -3071,6 +3107,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3226,6 +3265,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = await async_httpx_client.post( @@ -3239,7 +3289,13 @@ class BaseLLMHTTPHandler: if caller_requested_stream: if fake_stream is True: - return MockResponsesAPIStreamingIterator( + return replay_as_stream(response=response) + if not ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(await response.aread()) + ): + return ResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3249,17 +3305,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return ResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) await response.aread() else: response = await async_httpx_client.post( @@ -3276,6 +3321,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3493,12 +3541,20 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, + extra_query: Mapping[str, object] | None = None, extra_body: dict[str, object] | None = None, + stream: bool = False, + starting_after: int | None = None, + litellm_metadata: Mapping[str, object] = MappingProxyType({}), timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, shared_session: Optional["ClientSession"] = None, - ) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]: + ) -> ( + ResponsesAPIResponse + | SyncResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator] + ): """ Get a response by ID Uses GET /v1/responses/{response_id} endpoint in the responses API @@ -3511,7 +3567,11 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + extra_query=extra_query, extra_body=extra_body, + stream=stream, + starting_after=starting_after, + litellm_metadata=litellm_metadata, timeout=timeout, client=client, shared_session=shared_session, @@ -3540,20 +3600,23 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)} ## LOGGING logging_obj.pre_call( input="", api_key="", additional_args={ - "complete_input_dict": data, + "complete_input_dict": query, "api_base": api_base, "headers": headers, }, ) try: - response: Final = sync_httpx_client.get(url=url, headers=headers, params=data) + response: Final = sync_httpx_client.get( + url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None + ) response.raise_for_status() except Exception as e: raise self._handle_error( @@ -3561,6 +3624,20 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if stream and not _answered_json(response): + return SyncResponsesAPIStreamingIterator( + response=response, + model="", + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=dict(litellm_metadata), + custom_llm_provider=custom_llm_provider, + request_data={"litellm_params": dict(litellm_params)}, + replays_stored_stream=True, + ) + + if stream: + response.read() return responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, @@ -3574,11 +3651,15 @@ class BaseLLMHTTPHandler: logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, + extra_query: Mapping[str, object] | None = None, extra_body: dict[str, object] | None = None, + stream: bool = False, + starting_after: int | None = None, + litellm_metadata: Mapping[str, object] = MappingProxyType({}), timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, shared_session: Optional["ClientSession"] = None, - ) -> ResponsesAPIResponse: + ) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator: """ Async version of get_responses """ @@ -3616,20 +3697,23 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)} ## LOGGING logging_obj.pre_call( input="", api_key="", additional_args={ - "complete_input_dict": data, + "complete_input_dict": query, "api_base": api_base, "headers": headers, }, ) try: - response: Final = await async_httpx_client.get(url=url, headers=headers, params=data) + response: Final = await async_httpx_client.get( + url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None + ) response.raise_for_status() except Exception as e: verbose_logger.debug("Error retrieving response: %s", e) @@ -3638,6 +3722,20 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if stream and not _answered_json(response): + return ResponsesAPIStreamingIterator( + response=response, + model="", + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=dict(litellm_metadata), + custom_llm_provider=custom_llm_provider, + request_data={"litellm_params": dict(litellm_params)}, + replays_stored_stream=True, + ) + + if stream: + await response.aread() return responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, @@ -6182,9 +6280,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 48118c23ed7..60081c38073 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( handle_invalid_parallel_tool_calls, @@ -514,6 +515,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): self.update_optional_params_with_thinking_tokens( non_default_params=non_default_params, optional_params=mapped_params ) + gated_thinking: Final = strip_beta_gated_thinking( + mapped_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # mapped_params is the base config's bare dict + frozenset(), + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="databricks") + mapped_params["thinking"] = gated_thinking.thinking return mapped_params diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index e22ad85b5f9..90b63d860d3 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) ("content-type" if key.lower() == "content-type" else key): value for key, value in {**auth_headers, **client_anthropic_headers}.items() } - finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers + finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers headers=normalized_headers, optional_params=optional_params, messages=messages, @@ -191,6 +191,8 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + self._normalize_system_role_messages( # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict return anthropic_messages_request diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/github_copilot/authenticator.py b/litellm/llms/github_copilot/authenticator.py index 59e1804f453..ee0e7af7e9b 100644 --- a/litellm/llms/github_copilot/authenticator.py +++ b/litellm/llms/github_copilot/authenticator.py @@ -1,11 +1,12 @@ import json import os import time +from dataclasses import dataclass from datetime import datetime -from typing import Any, Final +from typing import Any, Final, TypeVar import httpx -from pydantic import ConfigDict, TypeAdapter, with_config +from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -13,7 +14,6 @@ from litellm.litellm_core_utils.asyncify import can_block_current_thread from litellm.llms.custom_httpx.http_handler import get_httpx_client from .common_utils import ( - APIKeyExpiredError, GetAccessTokenError, GetAPIKeyError, GetDeviceCodeError, @@ -27,6 +27,44 @@ DEFAULT_GITHUB_ACCESS_TOKEN_URL: Final = "https://github.com/login/oauth/access_ DEFAULT_GITHUB_API_KEY_URL: Final = "https://api.github.com/copilot_internal/v2/token" +class CopilotEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + api: str | None = None + + +class CopilotCachedKey(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + token: str + expires_at: float + endpoints: CopilotEndpoints | None = None + + +class CopilotCachedEndpoints(BaseModel): + model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True) + + endpoints: CopilotEndpoints | None = None + + +ApiKeyFileView = TypeVar("ApiKeyFileView", CopilotCachedKey, CopilotCachedEndpoints) + + +@dataclass(frozen=True, slots=True) +class _UnsavedApiKey: + access_token: str + key: CopilotCachedKey + + +_UNSAVED_API_KEYS: Final[dict[str, _UnsavedApiKey]] = {} # mutable-ok: refreshed keys api-key.json could not hold + + +def _read_token_file(path: str) -> str: + with open(path, "rb") as f: + raw: Final = f.read() + return raw.decode(json.detect_encoding(raw)) + + @with_config(ConfigDict(extra="allow", strict=True)) class _DeviceCode(TypedDict): device_code: ReadOnly[str] @@ -87,12 +125,13 @@ class Authenticator: GetAccessTokenError: If unable to obtain an access token after retries. """ try: - with open(self.access_token_file, "r") as f: - access_token = f.read().strip() - if access_token: - return access_token + saved_token: Final = _read_token_file(self.access_token_file).strip() + if saved_token: + return saved_token except OSError: verbose_logger.warning("No existing access token found or error reading file") + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a text access token: %s", self.access_token_file, e) if not can_block_current_thread(): raise GetAccessTokenError( @@ -135,64 +174,83 @@ class Authenticator: Raises: GetAPIKeyError: If unable to obtain an API key. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info = json.load(f) - if api_key_info.get("expires_at", 0) > datetime.now().timestamp(): - return api_key_info.get("token") - else: - verbose_logger.warning("API key expired, refreshing") - raise APIKeyExpiredError( - message="API key expired", - status_code=401, - ) - except OSError: - verbose_logger.warning("No API key file found or error opening file") - except (json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API key from file: %s", e) - except APIKeyExpiredError: - pass # Already logged in the try block + unsaved: Final = self._unsaved_api_key() + if unsaved is not None: + return unsaved.token + cached: Final = self._read_api_key_file(CopilotCachedKey) + if cached is not None and cached.expires_at > datetime.now().timestamp(): + return cached.token + if cached is not None: + verbose_logger.warning("API key expired, refreshing") try: - api_key_info = self._refresh_api_key() - with open(self.api_key_file, "w") as f: - json.dump(api_key_info, f) - token: Final = api_key_info.get("token") - if token: - return token - else: - raise GetAPIKeyError( - message="API key response missing token", - status_code=401, - ) - except OSError as e: - verbose_logger.error("Error saving API key to file: %s", e) - raise GetAPIKeyError( - message=f"Failed to save API key: {e}", - status_code=500, - ) + api_key_info: Final = self._refresh_api_key() except RefreshAPIKeyError as e: raise GetAPIKeyError( message=f"Failed to refresh API key: {e}", status_code=401, + ) from e + serialized: Final = json.dumps(api_key_info) + try: + with open(self.api_key_file, "w") as f: + f.write(serialized) + except OSError as e: + verbose_logger.warning( + "Keeping the refreshed API key in memory, it could not be saved to %s: %s", self.api_key_file, e ) + self._keep_unsaved_api_key(serialized) + token: Final = api_key_info.get("token") + if not token: + raise GetAPIKeyError( + message="API key response missing token", + status_code=401, + ) + return token def get_api_base(self) -> str | None: """ - Get the API endpoint from the api-key.json file. + Get the API endpoint of the current API key. Returns: Optional[str]: The GitHub Copilot API endpoint, or None if not found. """ - try: - with open(self.api_key_file, "r") as f: - api_key_info: Final = json.load(f) - endpoints: Final = api_key_info.get("endpoints", {}) - api_endpoint: Final = endpoints.get("api") - return api_endpoint - except (OSError, json.JSONDecodeError, KeyError) as e: - verbose_logger.warning("Error reading API endpoint from file: %s", e) + cached: Final = self._unsaved_api_key() or self._read_api_key_file(CopilotCachedEndpoints) + if cached is None or cached.endpoints is None: return None + return cached.endpoints.api + + def _read_api_key_file(self, view: type[ApiKeyFileView]) -> ApiKeyFileView | None: + try: + return view.model_validate_json(_read_token_file(self.api_key_file)) + except OSError: + verbose_logger.warning("No API key file found or error opening file") + return None + except ValueError as e: + verbose_logger.warning("Ignoring %s, not a Copilot API key file: %s", self.api_key_file, e) + return None + + def _stored_access_token(self) -> str | None: + try: + access_token: Final = _read_token_file(self.access_token_file).strip() + except (OSError, ValueError): + return None + return access_token or None + + def _unsaved_api_key(self) -> CopilotCachedKey | None: + unsaved: Final = _UNSAVED_API_KEYS.get(self.api_key_file) + if unsaved is None or unsaved.key.expires_at <= time.time(): + return None + return unsaved.key if unsaved.access_token == self._stored_access_token() else None + + def _keep_unsaved_api_key(self, serialized: str) -> None: + access_token: Final = self._stored_access_token() + if access_token is None: + return + try: + key: Final = CopilotCachedKey.model_validate_json(serialized) + except ValueError: + return + _UNSAVED_API_KEYS[self.api_key_file] = _UnsavedApiKey(access_token=access_token, key=key) def _refresh_api_key(self) -> dict[str, Any]: """ diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index ed06391a260..91832899141 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data. @@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 0311f6c6590..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -5,8 +5,10 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), + ), + "reasoning_effort", + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 12ead582975..a9d2dcdffb8 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: import litellm from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object -from litellm.litellm_core_utils.token_counter import token_counter +from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, @@ -104,6 +104,19 @@ def is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) +_OPENAI_AGENTS_API_RESOURCES: Final = frozenset({"agents", "vaults"}) + + +def openai_api_resource(path: str) -> str | None: + segments: Final = tuple(segment for segment in path.split("/") if segment) + resource_segments: Final = segments[1:] if segments[:1] == ("v1",) else segments + return next(iter(resource_segments), None) + + +def is_openai_agents_api_path(path: str) -> bool: + return openai_api_resource(path) in _OPENAI_AGENTS_API_RESOURCES + + class OpenAIError(BaseLLMException): def __init__( self, @@ -242,7 +255,7 @@ async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletio def build_output_token_limit_response( - e: openai.BadRequestError, data: Mapping[str, object], is_async: bool + e: openai.BadRequestError, data: Mapping[str, object], is_async: bool, base_model: str | None ) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]: """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets. @@ -251,7 +264,11 @@ def build_output_token_limit_response( """ model: Final[str] = str(data.get("model", "")) messages: Final = data.get("messages") - prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0 + prompt_tokens: Final = ( + token_counter(model=usage_tokenizer_model(model, base_model), messages=messages) + if isinstance(messages, list) + else 0 + ) if not data.get("stream"): return e.response.headers, _output_token_limit_completion(model, prompt_tokens) chunk: Final = _output_token_limit_chunk(model) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 9d4df354a30..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -44,12 +44,17 @@ from litellm.utils import ( CustomStreamWrapper, ProviderConfigManager, convert_to_model_response_object, + get_base_model_from_metadata, ) from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -205,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() @@ -557,7 +575,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=True) + return build_output_token_limit_response( + e=e, + data=data, + is_async=True, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: raise e @@ -604,7 +627,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.BadRequestError as e: if not is_output_token_limit_error(e): raise - return build_output_token_limit_response(e=e, data=data, is_async=False) + return build_output_token_limit_response( + e=e, + data=data, + is_async=False, + base_model=get_base_model_from_metadata(logging_obj.model_call_details), + ) except Exception as e: if raw_response is not None: raise Exception( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index b7e4cc6868f..5be488f0f6e 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl -from typing import TYPE_CHECKING, Final, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Literal, cast + +from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import AuthenticationError +from litellm.responses.streaming_iterator import map_stream_error_to_exception from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -25,6 +31,54 @@ from ..openai import OpenAIChatCompletion if TYPE_CHECKING: from fastapi import WebSocket + from openai.types.realtime import RealtimeError + from websockets.asyncio.client import ClientConnection + +_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" +_CREDENTIALS_REJECTED: Final = "invalid_api_key" + + +def first_event_error(first_event: str | bytes) -> "RealtimeError | None": + event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + from openai.types.realtime import RealtimeErrorEvent + + return RealtimeErrorEvent.model_validate(event).error + + +def _confirm_first_event( + first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + error: Final = first_event_error(first_event) + if error is None or ( + error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime + ): + return True + if error.code == _CREDENTIALS_REJECTED or not sent_credentials: + raise AuthenticationError(message=error.message, llm_provider="openai", model=model) + raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") + + +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime) + + +async def confirm_gateway_session_started( + connection: "ClientConnection", + model: str, + sent_credentials: bool, + model_listed_outside_realtime: bool, + first_event_wait_seconds: float, +) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds) + except asyncio.TimeoutError: + return True + return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime) class OpenAIRealtime(OpenAIChatCompletion): diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 45fc1fc3ff7..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + openai_prompt_cache_breakpoint_support, + targets_openai_api, +) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, + prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -27,6 +32,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_a from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -50,6 +56,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -66,6 +75,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"}) _COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) _COMPACT_REJECTED_PARAMS: Final = frozenset( { @@ -93,6 +103,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) +class _DeclaredModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + base_model: object = None + model_info: object = None + + class _DeploymentHost(BaseModel): """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`.""" @@ -216,6 +233,63 @@ def _with_breakpoint_on_last_reading_part( return (*parts[:last], marked_last, *parts[last + 1 :]) +def _json_object_or_none(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None + + +def _json_array_or_empty(value: object) -> Sequence[object]: + return value if isinstance(value, (list, tuple)) else () + + +def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(item) + if fields is None or not _carries_prompt_cache_breakpoint(fields): + return item + wire_safe_fields: Final = { + key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers) + for key, value in fields.items() + if key != "prompt_cache_breakpoint" + } + unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects( + wire_safe_fields.values(), fields.values() + ) + return item if unchanged else wire_safe_fields + + +def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool: + return all(new is old for new, old in zip(rebuilt, originals, strict=True)) + + +def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: + return "prompt_cache_breakpoint" in fields or any( + _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS + ) + + +def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool: + return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts)) + + +def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object: + if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value): + return value + parts: Final = _json_array_or_empty(value) + wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts) + return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts) + + +def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(part) + if fields is None or "prompt_cache_breakpoint" not in fields: + return part + marker: Final = fields["prompt_cache_breakpoint"] + wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None + if keeps_markers and wire_marker == marker: + return part + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, wire_marker) + + class _ReasoningSupportEntry(LiteLLMBaseModel): litellm_provider: str | None = None supports_reasoning: bool | None = None @@ -347,12 +421,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -394,17 +475,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -424,7 +508,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -442,7 +528,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -450,6 +536,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: @@ -544,12 +639,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): validated_input: Final = self._validate_input_param( self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) ) - marked_input: Final = ( - self._input_with_prompt_cache_breakpoints(validated_input) - if self._reads_prompt_cache_breakpoints( - model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options - ) - else validated_input + reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options + ) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints ) maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any( is_anthropic_tool_search_chat_tool(tool) for tool in tools or () @@ -559,20 +653,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if maps_anthropic_tool_search else tools ) - replayable_input: Final = ( - input_without_unanswered_anthropic_tool_search_calls(marked_input) - if maps_anthropic_tool_search - else marked_input - ) stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( - model=model, input=replayable_input, tools=mapped_tools + model=model, + input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), + tools=mapped_tools, ) + if marker_safe_input is not validated_input: + self._strip_caller_input_as_if_unmarked( + model=model, + caller_input=validated_input, + reads_breakpoints=reads_breakpoints, + maps_anthropic_tool_search=maps_anthropic_tool_search, + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + def _replayable_input( + self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool + ) -> str | ResponseInputParam: + marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input + if not maps_anthropic_tool_search: + return marked_input + return input_without_unanswered_anthropic_tool_search_calls(marked_input) + + def _strip_caller_input_as_if_unmarked( + self, + model: str, + caller_input: str | ResponseInputParam, + reads_breakpoints: bool, + maps_anthropic_tool_search: bool, + ) -> None: + self.remove_cache_control_flag_from_input_and_tools( + model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) + ) + + def _input_with_wire_safe_prompt_cache_breakpoints( + self, + model: str, + input: str | ResponseInputParam, + litellm_params: GenericLiteLLMParams, + reads_breakpoints: bool, + ) -> str | ResponseInputParam: + if not isinstance(input, list) or not drops_params(litellm_params): + return input + keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params + ) + wire_safe_items: Final = tuple( + _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input + ) + if _same_objects(wire_safe_items, input): + return input + wire_safe_input: Final = list(wire_safe_items) + return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker + + def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + provider: Final = _provider_name(self.custom_llm_provider) + base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + known_support: Final = ( + openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model)) + ) + return next((support for support in known_support if support is not None), None) is not False + def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url) @@ -688,14 +833,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _without_foreign_tool_call_item_id(item: object) -> object: - if not isinstance(item, dict): + fields: Final = _json_object_or_none(item) + if fields is None: return item - item_type: Final = item.get("type") - item_id: Final = item.get("id") + item_type: Final = fields.get("type") + item_id: Final = fields.get("id") genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix): return item - return {key: value for key, value in item.items() if key != "id"} + return {key: value for key, value in fields.items() if key != "id"} def _sanitized_tool_schemas_for_openai( self, @@ -739,6 +885,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None return base_model if isinstance(base_model, str) and base_model else model + @staticmethod + def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + declared: Final = _DeclaredModel.model_validate(litellm_params) + model_info: Final = _json_object_or_none(declared.model_info) + candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model")) + return next((name for name in candidates if isinstance(name, str) and name), model) + @staticmethod def _sanitized_tool_entry( entry: Mapping[str, object], @@ -1094,7 +1247,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: """ Transform the get response API request into a URL and data @@ -1103,7 +1256,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 8daae2a8e6a..a5d825a39d7 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def is_native_messages_route(self) -> bool: return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: + return False + def validate_anthropic_messages_environment( self, headers: dict[str, str], diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe88ac03e04..ad47b334ed4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -270,6 +270,13 @@ "/v1/responses" ] }, + "kyma": { + "base_url": "https://kymaapi.com/v1", + "api_key_env": "KYMA_API_KEY", + "provider_info": { + "supports_function_calling": true + } + }, "streamlake": { "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", "api_key_env": "STREAMLAKE_API_KEY", diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 5bff43828d6..884c82fe331 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from dataclasses import dataclass from typing import Final, Literal import httpx @@ -8,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import Cache, LiteLLMCacheType -from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -16,10 +16,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.llms.openai.openai import AllMessageValues -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.types.llms.openai import ChatCompletionToolParam +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url -from ..vertex_llm_base import VertexBase +from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, @@ -65,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) +_TOOL_PARAMS: Final = ("tools", "tool_choice") + + +def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler: + if timeout is None: + return HTTPHandler() + return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout) + + +class _CachedContentsNotServedError(VertexAIError): + pass + + def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None: tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0 logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count @@ -78,6 +92,61 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: return timeout +@dataclass(frozen=True, slots=True) +class _CacheTarget: + model: str + project: str | None + location: str | None + + +def _cache_target( + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + api_base: str | None, + model: str, + vertex_project: str | None, + vertex_location: str | None, +) -> _CacheTarget: + model_location: Final = ( + None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base) + ) + if model_location is None: + return _CacheTarget(model=model, project=vertex_project, location=vertex_location) + return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location) + + +def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int: + return max( + get_prompt_cache_min_tokens( + model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}" + ) + for model in models + ) + + +def _context_caching_skip_reason( + cached_messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam] | None, + deployment_model: str, + cache_model: str, + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], +) -> str | None: + if not cached_messages_end_on_supported_turn(cached_messages): + return ( + "Vertex AI context caching: cached message block ends on a model turn once " + "system messages are extracted, which the cachedContents API rejects." + ) + min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider) + if not is_prompt_caching_valid_prompt( + model=cache_model, + messages=list(cached_messages), + tools=None if tools is None else list(tools), + custom_llm_provider=custom_llm_provider, + min_token_count=min_tokens, + ): + return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})." + return None + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -135,6 +204,7 @@ class ContextCachingEndpoints(VertexBase): vertex_project=vertex_project, vertex_location=vertex_location, vertex_api_version=("v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1"), + collection_endpoint=True, ) def check_cache( @@ -206,6 +276,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -305,6 +377,8 @@ class ContextCachingEndpoints(VertexBase): except httpx.HTTPStatusError as e: if e.response.status_code == 403: return None + if e.response.status_code == 404 and custom_llm_provider == "gemini": + raise _CachedContentsNotServedError(status_code=404, message=e.response.text) raise VertexAIError(status_code=e.response.status_code, message=e.response.text) except Exception as e: raise VertexAIError(status_code=500, message=str(e)) @@ -370,41 +444,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -417,44 +475,42 @@ class ContextCachingEndpoints(VertexBase): if extra_headers is not None: headers.update(extra_headers) - if client is None or not isinstance(client, HTTPHandler): - _params: Final = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = HTTPHandler(**_params) - else: - client = client + http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout) ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model - ) - google_cache_name: Final = self.check_cache( - cache_key=generated_cache_key, - client=client, - headers=headers, - api_key=api_key, - api_base=api_base, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_auth_header=vertex_auth_header, - model=model, + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) + try: + google_cache_name: Final = self.check_cache( + cache_key=generated_cache_key, + client=http_client, + headers=headers, + api_key=api_key, + api_base=api_base, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vertex_project=target.project, + vertex_location=target.location, + vertex_auth_header=vertex_auth_header, + model=target.model, + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) if google_cache_name: return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -473,7 +529,7 @@ class ContextCachingEndpoints(VertexBase): ) try: - response: Final = client.post( + response: Final = http_client.post( url=url, headers=headers, json=cached_content_request_body, @@ -489,7 +545,7 @@ class ContextCachingEndpoints(VertexBase): _record_cache_creation_tokens(logging_obj, raw_response_cached) self._known_caches.remember( KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ), generated_cache_key, raw_response_cached.get("name"), @@ -536,41 +592,25 @@ class ContextCachingEndpoints(VertexBase): if len(cached_messages) == 0: return messages, optional_params, None - if not cached_messages_end_on_supported_turn(cached_messages): - verbose_logger.debug( - "Vertex AI context caching: cached message block ends on a model turn once " - "system messages are extracted, which the cachedContents API rejects. " - "Skipping context caching." - ) + target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location) + tools: Final = optional_params.get("tools") + tool_choice: Final = optional_params.get("tool_choice") + skip_reason: Final = _context_caching_skip_reason( + cached_messages, tools, model, target.model, custom_llm_provider + ) + if skip_reason is not None: + verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None - # Gemini requires a minimum of 1024 tokens for context caching. - # Skip caching if the cached content is too small to avoid API errors. - if not is_prompt_caching_valid_prompt( - model=model, - messages=cached_messages, - tools=optional_params.get("tools"), - custom_llm_provider=custom_llm_provider, - ): - verbose_logger.debug( - "Vertex AI context caching: cached content is below minimum token " - "count (%d). Skipping context caching.", - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, - ) - return messages, optional_params, None - - tools: Final = optional_params.pop("tools", None) - tool_choice: Final = optional_params.pop("tool_choice", None) - ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) headers: Final = { @@ -590,10 +630,10 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key: Final = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model ) scope: Final = KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location ) async def lookup_or_create() -> str | None: @@ -605,10 +645,10 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, vertex_auth_header=vertex_auth_header, - model=model, + model=target.model, ) if google_cache_name: @@ -616,12 +656,12 @@ class ContextCachingEndpoints(VertexBase): ## TRANSFORM REQUEST cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, + model=target.model, messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=target.project, + vertex_location=target.location, ) cached_content_request_body["tools"] = tools @@ -659,9 +699,15 @@ class ContextCachingEndpoints(VertexBase): ) return raw_response_cached.get("name") - cache_name: Final = await self._known_caches.single_flight( - scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) - ) + try: + cache_name: Final = await self._known_caches.single_flight( + scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) + ) + except _CachedContentsNotServedError: + verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name def get_cache(self): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..bfa94396ea7 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -211,6 +211,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + beta_values.update( + anthropic_model_info.thinking_betas( + optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict + ) + ) + if beta_values: request_headers["anthropic-beta"] = ",".join(beta_values) @@ -245,7 +251,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 363e5596cb9..79fd9f9e8b1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -145,6 +145,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="vertex_ai", + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) beta_set: Final = set(auto_betas) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 4be41e26326..180761f15d9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,9 +6,12 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ +from collections.abc import Mapping from typing import Any, Final +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -20,6 +23,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +def count_tokens_request_body( + request_data: Mapping[str, object], model: str, drop_params: object +) -> Mapping[str, object]: + tools: Final = request_data.get("tools") + if not isinstance(tools, list): + return request_data + return { + **request_data, + "tools": tools_without_custom_field( + OBJECT_LIST_ADAPTER.validate_python(tools), + hoist_defer_loading=is_claude_model_target("vertex_ai", model), + drop_custom=drops_unsupported_params(drop_params), + ), + } + + def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) @@ -169,7 +188,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): response: Final = await async_client.post( endpoint_url, headers=headers, - json=request_data, + json=dict(count_tokens_request_body(request_data, model, litellm_params.get("drop_params"))), timeout=30.0, ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 0cdd92bb071..7dd98212f76 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -6,9 +6,11 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import asyncio import json +import re import threading from collections import OrderedDict from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse @@ -46,6 +48,31 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str: _graft_default_vertex_path = graft_default_vertex_path + +_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)") + + +@dataclass(frozen=True, slots=True) +class PublisherModelLocation: + parent_url: str + project: str + location: str + model: str + + +def publisher_model_location(api_base: str) -> PublisherModelLocation | None: + match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base) + if match is None: + return None + _, _, project, _, location = match.group().rsplit("/", 4) + return PublisherModelLocation( + parent_url=match.group(), + project=project, + location=location, + model=api_base.rstrip("/").rsplit("/", 1)[-1], + ) + + GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform" ) @@ -637,15 +664,18 @@ class VertexBase: vertex_location: str | None = None, vertex_api_version: Literal["v1", "v1beta1"] | None = None, use_psc_endpoint_format: bool = False, + collection_endpoint: bool = False, ) -> tuple[str | None, str]: """ for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317 Handles custom api_base for: - 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} + 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}, or {api_base}/{endpoint} + for a collection endpoint 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the api_base when its path is empty or only an API version (/v1, /v1beta1); - otherwise constructs {api_base}:{endpoint} + otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base + naming a publisher model, the collection under that model's projects/{project}/locations/{location} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -653,22 +683,22 @@ class VertexBase: Args: use_psc_endpoint_format: If True, constructs PSC endpoint URL format. If False (default), uses api_base as-is and appends :{endpoint} + collection_endpoint: True when `endpoint` names a collection resource such as + cachedContents rather than a model action such as generateContent ## Returns - (auth_header, url) - Tuple[Optional[str], str] """ if api_base: if custom_llm_provider == "gemini": - # For Gemini (Google AI Studio), construct the full path like other providers - if model is None: + if model is None and not collection_endpoint: raise ValueError("Model parameter is required for Gemini custom API base URLs") - url = join_api_base(api_base, f"models/{model}:{endpoint}") + url = join_api_base(api_base, endpoint if collection_endpoint else f"models/{model}:{endpoint}") if gemini_api_key is None: raise ValueError( "Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable." ) - if gemini_api_key is not None: - auth_header = {"x-goog-api-key": gemini_api_key} + auth_header = {"x-goog-api-key": gemini_api_key} else: # For Vertex AI if use_psc_endpoint_format: @@ -692,7 +722,12 @@ class VertexBase: elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: - url = f"{api_base.rstrip('/')}:{endpoint}" + model_location: Final = publisher_model_location(api_base) if collection_endpoint else None + url = ( + f"{api_base.rstrip('/')}:{endpoint}" + if model_location is None + else join_api_base(model_location.parent_url, endpoint) + ) if stream is True: parsed_stream_url: Final = urlparse(url) stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 4076ed06fc2..7f2425d0909 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -313,10 +313,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[str, dict]: + ) -> tuple[str, dict[str, str]]: encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") url: Final = f"{api_base}/{encoded_response_id}" - data: Final[dict] = {} + data: Final[dict[str, str]] = {} return url, data def transform_get_response_api_response( diff --git a/litellm/main.py b/litellm/main.py index 41c4302499e..d1af2de3a13 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -155,6 +155,7 @@ from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + AgenticLoopParams, BaseUrlRoutingHint, CustomPricingLiteLLMParams, ModelResponseStream, @@ -234,6 +235,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export ) from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.anthropic.common_utils import is_claude_tool_target from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -1135,17 +1137,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o _ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) -def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: - if custom_llm_provider == "anthropic": - return True - model_lower: Final = model.lower() - if custom_llm_provider == "bedrock": - return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower) - if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai": - return "claude" in model_lower - return False - - def _history_without_unanswered_tool_search_calls( messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig ) -> tuple[object, ...] | None: @@ -1282,6 +1273,16 @@ def _get_router_deployment_id(kwargs: dict) -> str | None: return None +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + def _register_custom_pricing_for_request( model: str, custom_llm_provider: str, @@ -2672,10 +2673,13 @@ def _complete_custom_openai( # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there api_base = _resolve_openai_api_base(api_base) + if custom_llm_provider == "bigmodel" and not api_key: + raise litellm.AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) organization = ( organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") + or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION")) or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) openai.organization = organization @@ -5698,6 +5702,7 @@ def completion( **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, "extra_headers": extra_headers, } + requested_model: Final = model model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5709,16 +5714,23 @@ def completion( else None ), ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5749,7 +5761,7 @@ def completion( } ) - if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): + if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) if provider_specific_header is not None: @@ -5815,7 +5827,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) @@ -6299,6 +6319,7 @@ def completion( or custom_llm_provider == "nebius" or custom_llm_provider == "wandb" or custom_llm_provider == "clarifai" + or custom_llm_provider == "bigmodel" or custom_llm_provider in litellm.openai_compatible_providers or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 0e2fc2e3dcb..8e6becc60b4 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -111,6 +111,7 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, + Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field, } ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 23c6ccf9c0a..4fad4857c60 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28071,7 +28071,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -28345,7 +28345,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -78339,6 +78341,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, @@ -78995,5 +80245,81 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index e40cd134759..4188866de08 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -5326,6 +5326,58 @@ "a2a": false, "generateContent": false } + }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } } }, "endpoints": { diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 39adb99a0d0..dd964a74735 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7816,7 +7816,7 @@ "paths": { "/claude-code/marketplace.json": { "get": { - "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout a key the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nWith `general_settings.claude_code_marketplace_auth_required: true` the key must come\nin the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the\nmarketplace source's `headers`), `?key=` is ignored, and a caller with no valid key\ngets a 401 with `WWW-Authenticate: Bearer`.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", "operationId": "get_marketplace_claude_code_marketplace_json_get", "parameters": [ { @@ -33639,7 +33639,7 @@ ] }, "get": { - "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```", + "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Resume a background response's event stream after sequence number 7\ncurl -N \"http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```", "operationId": "get_response_openai_v1_responses__response_id__get", "parameters": [ { @@ -33650,6 +33650,46 @@ "title": "Response Id", "type": "string" } + }, + { + "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)", + "in": "query", + "name": "stream", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "description": "With stream=true, replay only the events after this sequence number", + "in": "query", + "name": "starting_after", + "required": false, + "schema": { + "minimum": 0, + "type": "integer" + } + }, + { + "description": "Additional output data to include, forwarded to the provider", + "in": "query", + "name": "include", + "required": false, + "schema": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + { + "description": "With stream=true, whether delta events carry obfuscation padding", + "in": "query", + "name": "include_obfuscation", + "required": false, + "schema": { + "type": "boolean" + } } ], "responses": { @@ -33659,6 +33699,12 @@ "schema": { "$ref": "#/components/schemas/ResponsesAPIResponse" } + }, + "text/event-stream": { + "schema": { + "description": "Server sent events when stream=true", + "type": "string" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index df139a9b84d..4047fa3f0ad 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + claude_code_marketplace_auth_required: bool = Field( + False, + description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.", + ) disable_requester_ip_address_logging: bool | None = Field( None, description="If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.", @@ -3744,6 +3748,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index b0088d93112..fc82686947d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -7,7 +7,8 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on any HTTPS host (S3, Artifactory, a static file server). Endpoints: -/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills) +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (anonymous unless + general_settings.claude_code_marketplace_auth_required; a key adds its granted skills) /claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only) /claude-code/plugins - GET - List plugins visible to the key (enabled, plus granted disabled ones) /claude-code/plugins/{name} - GET - Get plugin details (403 on a disabled plugin the key is not granted) @@ -31,14 +32,17 @@ from pydantic import TypeAdapter from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( SkillVisibility, resolve_skill_visibility, ) +from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository +from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( InstallationPreference, ListPluginsResponse, @@ -109,6 +113,41 @@ async def get_prisma_client() -> object: _get_prisma_client: Final = get_prisma_client +async def _header_key_caller(request: Request) -> UserAPIKeyAuth: + try: + caller: Final = await user_api_key_auth( + request=request, + api_key=request.headers.get(SpecialHeaders.openai_authorization.value) or "", + custom_litellm_key_header=request.headers.get(SpecialHeaders.custom_litellm_api_key.value), + ) + except ProxyException as e: + if e.code != "401": + raise + raise ProxyException( + message=e.message, + type=e.type, + param=e.param, + code=401, + headers={**e.headers, "WWW-Authenticate": "Bearer"}, + ) from e + if caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + raise ProxyException( + message="The LiteLLM key could not be verified because the database is unavailable", + type=ProxyErrorTypes.no_db_connection, + param=None, + code=503, + ) + return caller + + +async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAuth | None: + if marketplace_auth_required(): + return await _header_key_caller(request) + if key: + return await user_api_key_auth(request=request, api_key=f"Bearer {key}") + return None + + @router.get( "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], @@ -121,10 +160,15 @@ async def get_marketplace(request: Request, key: str | None = None): - claude plugin marketplace add - claude plugin install @ - Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + Without a key the catalog holds the enabled (public) plugins. With `?key=sk-...` the key is authenticated and the catalog also holds the disabled plugins granted to it through `object_permission.skills` or an access group on the key or its team. + With `general_settings.claude_code_marketplace_auth_required: true` the key must come + in the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the + marketplace source's `headers`), `?key=` is ignored, and a caller with no valid key + gets a 401 with `WWW-Authenticate: Bearer`. + Returns: Marketplace catalog with list of available plugins and their git sources. @@ -137,10 +181,7 @@ async def get_marketplace(request: Request, key: str | None = None): """ try: prisma_client: Final = await get_prisma_client() - - caller: Final[UserAPIKeyAuth | None] = ( - await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None - ) + caller: Final = await _marketplace_caller(request, key) visibility: Final[SkillVisibility] = await resolve_skill_visibility(caller) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( where=visibility.where() @@ -201,7 +242,7 @@ async def get_marketplace(request: Request, key: str | None = None): verbose_proxy_logger.exception("Error generating marketplace: %s", e) raise HTTPException( status_code=500, - detail={"error": f"Failed to generate marketplace: {e}"}, + detail={"error": "Failed to generate marketplace"}, ) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..1a2e610d5c4 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -2,7 +2,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) """ -from collections.abc import Mapping +from collections.abc import AsyncGenerator, Mapping from types import MappingProxyType from typing import Final @@ -19,12 +19,10 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) -from litellm.llms.base_llm.guardrail_translation.utils import ( - blocked_response_usage as _blocked_response_usage, -) from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( @@ -44,6 +42,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, error_status_code, + litellm_call_id_headers, openai_error_param, openai_error_type, with_litellm_call_id, @@ -58,6 +57,8 @@ from litellm.router import Router router: Final = APIRouter() +_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: if not exc.provider_specific_fields: @@ -229,44 +230,22 @@ async def anthropic_response( request_data=_data, ) - # Create Anthropic-formatted response with violation message - import uuid + block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS + block_handler: Final = AnthropicMessagesHandler() - from litellm.types.utils import AnthropicMessagesResponse + if data.get("stream") is True: + block_chunks: Final = block_handler.build_block_sse_chunks(e) - # Report the blocked LLM response's real token usage (carried on the - # exception) instead of discarding it; zero for pre-call blocks. - _usage: Final = _blocked_response_usage(e.original_response) - - _anthropic_response: Final = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()}", - type="message", - role="assistant", - content=[{"type": "text", "text": e.message}], - model=e.model, - stop_reason="end_turn", - usage=_usage, - ) - - if data.get("stream", None) is not None and data["stream"] is True: - # For streaming, use the standard SSE data generator - async def _passthrough_stream_generator(): - yield _anthropic_response - - selected_data_generator: Final = ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=_passthrough_stream_generator(), - user_api_key_dict=user_api_key_dict, - request_data=_data, - proxy_logging_obj=proxy_logging_obj, - ) + async def _blocked_stream() -> AsyncGenerator[str, None]: + for chunk in block_chunks: + yield chunk.decode() return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers={}, + generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers ) - return _anthropic_response + fastapi_response.headers.update(block_headers) + return block_handler.build_block_response(e) except AnthropicContextManagementError as e: if e.status_code >= 500: # Server-side polyfill failures hit the failure hook for spend/alert diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index 1911cb574ef..6c58f2edc02 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -45,7 +45,11 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_set_request_parsed_body, ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET -from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.management_endpoints.ui_sso import ( + CLI_SSO_BROWSER_TEAM_CHOICE, + CLI_SSO_CHOSEN_TEAM_ID, + CliSsoTeamDetail, +) from litellm.proxy.mcp_server.gateway_dcr_flow import ( PROXY_API_AUDIENCE, MintedProxyCredential, @@ -235,6 +239,7 @@ async def device_authorization(request: Request) -> JSONResponse: "sso_complete": False, "user_code_verified": False, "session_data": None, + CLI_SSO_BROWSER_TEAM_CHOICE: True, } _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) @@ -267,7 +272,15 @@ def _validate_login(flow: Mapping[str, object]) -> _GatewayLogin | _OAuthError: status_code=400, error="invalid_grant", description="The login session is malformed; sign in again" ) - team_id: Final = session_data.teams[0] if session_data.teams else None + chosen: Final = flow.get(CLI_SSO_CHOSEN_TEAM_ID) + chosen_team_id: Final = chosen if isinstance(chosen, str) else None + if chosen_team_id is not None and chosen_team_id not in session_data.teams: + return _OAuthError( + status_code=400, + error="invalid_grant", + description="The team picked at sign-in is no longer one of your teams; sign in again", + ) + team_id: Final = chosen_team_id or next(iter(session_data.teams), None) selected_team: Final = selected_cli_sso_team_detail(team_details=session_data.team_details, team_id=team_id) if selected_team is None: return _OAuthError( diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..48884d0fedc 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -20,6 +20,8 @@ from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models +LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) + _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -458,23 +460,21 @@ def get_known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - known_providers: Final = {provider.value for provider in LlmProviders} - suffix_appended_wildcard_models: Final = [] - for model in wildcard_models: - if not model.startswith(f"{wildcard_provider_prefix}/"): - # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b"). - # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish - # multiple instances), replace that existing provider prefix instead of stacking - # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". - # Only strip the leading segment when it is a known provider, so ids whose first - # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. - leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: - model = f"{wildcard_provider_prefix}/{model_suffix}" - else: - model = f"{wildcard_provider_prefix}/{model}" - suffix_appended_wildcard_models.append(model) - return list(dict.fromkeys(suffix_appended_wildcard_models)) + return list( + dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) + ) + + +def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str: + if model.startswith(f"{wildcard_provider_prefix}/"): + return model + leading, sep, model_suffix = model.partition("/") + leading_is_listed_providers_own_prefix: Final = ( + sep != "" + and leading in LLM_PROVIDER_PREFIXES + and (leading == listed_provider or listed_provider.startswith(f"{leading}_")) + ) + return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}" def expand_wildcard_deployments_for_model_info( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 95c31cbe642..2eaa647ac52 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3209,7 +3209,7 @@ class ProxyBaseLLMRequestProcessing: request_data=self.data, request=request, ) - if route_type == "aresponses": + if route_type in ("aresponses", "aget_responses"): selected_data_generator = ( ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( original_stream_response=response, diff --git a/litellm/proxy/common_utils/claude_code_marketplace_auth.py b/litellm/proxy/common_utils/claude_code_marketplace_auth.py new file mode 100644 index 00000000000..0cfb3061744 --- /dev/null +++ b/litellm/proxy/common_utils/claude_code_marketplace_auth.py @@ -0,0 +1,17 @@ +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +_AUTH_REQUIRED_SETTING: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None) + + +def marketplace_auth_required() -> bool: + from litellm.proxy.proxy_server import general_settings_view + + try: + return ( + _AUTH_REQUIRED_SETTING.validate_python(general_settings_view().get("claude_code_marketplace_auth_required")) + is True + ) + except ValidationError: + return True diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 397523288cf..47cd1db9f39 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -102,6 +102,7 @@ class CodexFallbackModel(LiteLLMBaseModel): experimental_supported_tools: tuple[()] = () context_window: int | None base_instructions: str + supports_search_tool: bool = False class CodexStockUpgrade(LiteLLMBaseModel): @@ -134,6 +135,7 @@ class CodexStockModel(LiteLLMBaseModel): upgrade: CodexStockUpgrade | None = None service_tiers: tuple[CodexServiceTier, ...] = () default_service_tier: str | None = None + supports_search_tool: bool = False supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = () context_window: int | None = None @@ -160,6 +162,7 @@ class CodexCatalogRow: reasoning_effort: str | None = None display_name: str | None = None service_tiers: tuple[object, ...] = () + search_tool_support: tuple[bool | None, ...] = () @cache @@ -252,6 +255,12 @@ def configured_service_tiers( return offered +def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None: + if False in per_deployment: + return False + return True if per_deployment and None not in per_deployment else None + + def _codex_can_drive(row: CodexCatalogRow) -> bool: return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES) @@ -306,6 +315,7 @@ def _entry( {tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())} ) configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers) + search_tool: Final = offered_search_tool(row.search_tool_support) if stock_model is None: return CodexFallbackModel( slug=row.id, @@ -314,6 +324,7 @@ def _entry( service_tiers=configured if configured is not None else (), context_window=row.max_input_tokens, base_instructions=instructions, + supports_search_tool=search_tool is True, ) upgrade: Final = ( stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None @@ -327,6 +338,7 @@ def _entry( "visibility": "list", "supported_in_api": True, "upgrade": upgrade, + **({} if search_tool is None else {"supports_search_tool": search_tool}), **(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}), } if configured is None: @@ -433,6 +445,7 @@ def _catalog_row( upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None, display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None, service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (), + search_tool_support=llm_router.get_search_tool_support(lookup_id, team_id) if llm_router is not None else (), ) diff --git a/litellm/proxy/common_utils/html_forms/native_client_consent.py b/litellm/proxy/common_utils/html_forms/native_client_consent.py index dac92c4e787..aeadc93221c 100644 --- a/litellm/proxy/common_utils/html_forms/native_client_consent.py +++ b/litellm/proxy/common_utils/html_forms/native_client_consent.py @@ -62,7 +62,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p

Approving issues it a personal credential that expires within {CLI_JWT_EXPIRATION_HOURS} hours. lite logout stops it from being renewed. Only approve if you started this sign-in yourself.

-{_team_field(teams)} +{team_select_field(teams)}
@@ -74,7 +74,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p """ -def _team_field(teams: Sequence[tuple[str, str]]) -> str: +def team_select_field(teams: Sequence[tuple[str, str]], *, require_choice: bool = False) -> str: if not teams: return "" if len(teams) == 1: @@ -83,9 +83,12 @@ def _team_field(teams: Sequence[tuple[str, str]]) -> str: f'' f"

Requests are attributed to team {escape(team_label)}.

" ) + placeholder: Final = '' if require_choice else "" options: Final = "".join( f'' for team_id, team_label in teams ) + required: Final = " required" if require_choice else "" return ( - f'' + '' + f'' ) diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 83cc4501466..d126f112f90 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -4,7 +4,7 @@ import os import subprocess from typing import Final -from litellm._logging import print_startup_message, verbose_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_logger def extract_sql_commands(diff_output: str) -> list[str]: @@ -63,7 +63,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: run_prisma, ) except ImportError as e: - print_startup_message( + print_startup_warning( f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}", ) return False, [] @@ -96,14 +96,14 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: else: return False, [] except subprocess.TimeoutExpired: - print_startup_message( + print_startup_warning( f"Timed out after {timeout}s generating the migration diff. " f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer.", ) return False, [] except subprocess.CalledProcessError as e: error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}" - print_startup_message(error_message) + print_startup_warning(error_message) return False, [] diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 7f5efcdaec3..170eb450339 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -421,7 +421,10 @@ class SpendLogCleanup: general_settings=None, redis_cache: RedisCache | None = None, partition_manager: SpendLogsPartitionManager | None = None, + *, + clock: Callable[[], float] = time.monotonic, ): + self._clock = clock self.retention_seconds: int | None = None self.partition_manager = partition_manager or SpendLogsPartitionManager() from litellm.proxy.proxy_server import general_settings as default_settings @@ -586,7 +589,7 @@ class SpendLogCleanup: async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None: """Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -624,7 +627,7 @@ class SpendLogCleanup: stale.allocated_bytes, _ANALYZE_SQL, ) - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -653,7 +656,7 @@ class SpendLogCleanup: async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None: """The reclaimable count saved by the last check, or None on the first check or when it cannot be read.""" - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) try: @@ -686,7 +689,7 @@ class SpendLogCleanup: the tool-index rows that expire with their spend logs. When fewer rows than that remain, the step is everything up to now. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return None timeout_ms: Final = self._timeout_ms(deadline) async with db_span("size_step_cutoff", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx: @@ -718,11 +721,11 @@ class SpendLogCleanup: Interpolating this into SQL is safe by construction: an int cannot carry SQL, and SET does not accept a bind parameter. """ - remaining_ms: Final = int((deadline - time.monotonic()) * 1000) + remaining_ms: Final = int((deadline - self._clock()) * 1000) return max(1, min(int(self.batch_timeout_seconds * 1000), remaining_ms)) @staticmethod - def _group_deadline(overall_deadline: float, groups_remaining: int) -> float: + def _group_deadline(overall_deadline: float, groups_remaining: int, now: float) -> float: """ Give each pending cleanup group an equal share of the time left. @@ -731,10 +734,9 @@ class SpendLogCleanup: """ if groups_remaining == 1: return overall_deadline - current_time: Final = time.monotonic() - if current_time >= overall_deadline: + if now >= overall_deadline: return overall_deadline - return current_time + (overall_deadline - current_time) / groups_remaining + return now + (overall_deadline - now) / groups_remaining def _remaining_timeout_ms(self, deadline: float) -> RemainingTimeoutMs: """ @@ -748,7 +750,7 @@ class SpendLogCleanup: """ def remaining() -> int | None: - return None if time.monotonic() >= deadline else self._timeout_ms(deadline) + return None if self._clock() >= deadline else self._timeout_ms(deadline) return remaining @@ -841,7 +843,7 @@ class SpendLogCleanup: run_count = 0 consecutive_failures = 0 while True: - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info( "Run budget exhausted during %s cleanup after %d rows; the next run resumes from here", table_name, @@ -858,13 +860,13 @@ class SpendLogCleanup: prisma_client, cutoff_date, table_name, time_column, total_deleted, "batch_cap_reached", deadline ) # Find rows and delete them in one go without fetching to application - batch_started_at = time.monotonic() + batch_started_at = self._clock() try: batch_result = await self._execute_delete_batch( prisma_client, delete_sql, cutoff_date, table_name, deadline ) except Exception as batch_exc: - if time.monotonic() >= deadline: + if self._clock() >= deadline: # The statement timeout was clamped to the budget that was # left, so this batch was cancelled by the deadline itself. # That is the bound working, not a database fault, and @@ -920,7 +922,7 @@ class SpendLogCleanup: consecutive_failures = 0 deleted_count = batch_result - SpendLogCleanupMetrics.record_batch(table_name, deleted_count, time.monotonic() - batch_started_at) + SpendLogCleanupMetrics.record_batch(table_name, deleted_count, self._clock() - batch_started_at) verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name) if deleted_count == 0: @@ -957,7 +959,7 @@ class SpendLogCleanup: operator the backlog was not drained; the gauge simply keeps its value from the last run that finished inside its budget. """ - if time.monotonic() >= deadline: + if self._clock() >= deadline: return TableCleanupResult(table_name=table_name, rows_deleted=rows_deleted, stop_reason=stop_reason) remaining: Final = await self._count_remaining(prisma_client, cutoff_date, table_name, time_column, deadline) if remaining is not None: @@ -1070,7 +1072,7 @@ class SpendLogCleanup: # behind any long-running reader for as long as that reader lives, which # is the one way this job could still outlast its budget without bound. remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline: + if self._clock() >= deadline: verbose_proxy_logger.info("Run budget already spent, skipping partition maintenance this run") elif self.general_settings.get( "use_spend_logs_partitioning", False @@ -1082,7 +1084,7 @@ class SpendLogCleanup: verbose_proxy_logger.info("Dropped %d expired spend-log partitions: %s", len(dropped), dropped) logs_result: Final = await self._delete_old_logs( - prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2) + prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2, now=self._clock()) ) verbose_proxy_logger.info("Deleted %s logs", logs_result.rows_deleted) @@ -1210,7 +1212,7 @@ class SpendLogCleanup: self, prisma_client: PrismaClient, deadline: float, still_over_target: Callable[[], Awaitable[bool]] ) -> tuple[str, ...]: remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline) - if time.monotonic() >= deadline or not self._setting("use_spend_logs_partitioning"): + if self._clock() >= deadline or not self._setting("use_spend_logs_partitioning"): return () if not await self.partition_manager.is_partitioned(prisma_client, remaining_timeout_ms): return () @@ -1257,7 +1259,7 @@ class SpendLogCleanup: needed: Final = rows_still_needed(state.logs_deleted + state.index_deleted) if needed <= 0: return state.stopped("target_reached") - if time.monotonic() >= deadline: + if self._clock() >= deadline: return state.stopped("budget_exhausted") if state.batches_left <= 0: return state.stopped("batch_cap_reached") @@ -1318,11 +1320,11 @@ class SpendLogCleanup: except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job verbose_proxy_logger.warning("Auto-router baseline retention remains pending") sessions_result: Final = await self._delete_old_autorouter_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 3) + prisma_client, session_cutoff, self._group_deadline(deadline, 3, self._clock()) ) verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted) user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows( - prisma_client, session_cutoff, self._group_deadline(deadline, 2) + prisma_client, session_cutoff, self._group_deadline(deadline, 2, self._clock()) ) verbose_proxy_logger.info( "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted @@ -1443,7 +1445,7 @@ class SpendLogCleanup: async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None: lock_acquired = False - run_started_at: Final = time.monotonic() + run_started_at: Final = self._clock() progress: Final = _RunProgress() progress_token: Final = _run_progress.set(progress) try: @@ -1493,7 +1495,7 @@ class SpendLogCleanup: SpendLogCleanupMetrics.record_run("skipped_locked") return - deadline: Final = time.monotonic() + self.run_budget_seconds + deadline: Final = self._clock() + self.run_budget_seconds size_group: Final = int(active_size_rule is not None) configured_group_count: Final = ( int(delete_spend_logs and self.retention_seconds is not None) @@ -1506,7 +1508,7 @@ class SpendLogCleanup: spend_log_results: Final = ( await self._clean_spend_log_tables( prisma_client, - self._group_deadline(deadline, configured_group_count), + self._group_deadline(deadline, configured_group_count, self._clock()), ) if delete_spend_logs and self.retention_seconds is not None else () @@ -1526,7 +1528,7 @@ class SpendLogCleanup: await self._clean_session_rollup( prisma_client, autorouter_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_spend_logs), + self._group_deadline(deadline, remaining_groups_after_spend_logs, self._clock()), ) if autorouter_retention_seconds is not None else () @@ -1535,7 +1537,7 @@ class SpendLogCleanup: await self._clean_health_checks( prisma_client, health_check_retention_seconds, - self._group_deadline(deadline, remaining_groups_after_sessions), + self._group_deadline(deadline, remaining_groups_after_sessions, self._clock()), ) if health_check_retention_seconds is not None else () @@ -1544,7 +1546,7 @@ class SpendLogCleanup: await self._clean_daily_tag_spend( prisma_client, daily_tag_spend_retention_seconds, - self._group_deadline(deadline, 1 + size_group), + self._group_deadline(deadline, 1 + size_group, self._clock()), ) if daily_tag_spend_retention_seconds is not None else () @@ -1561,12 +1563,12 @@ class SpendLogCleanup: ) outcome: Final = self._run_outcome(results) SpendLogCleanupMetrics.record_run(outcome) - self._log_run_summary(outcome, results, time.monotonic() - run_started_at, size_report) + self._log_run_summary(outcome, results, self._clock() - run_started_at, size_report) except asyncio.CancelledError: verbose_proxy_logger.error( "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here", - time.monotonic() - run_started_at, + self._clock() - run_started_at, progress.rows_deleted, progress.batches, ) diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 792ef8a3f8c..36f424507d7 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter +from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required from litellm.proxy.common_utils.html_forms.default_credentials_hint import should_hide_default_credentials_hint from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_enabled from litellm.types.proxy.discovery_endpoints.ui_discovery_endpoints import ( @@ -51,4 +52,5 @@ async def get_ui_config(): workers=proxy_config.worker_registry if is_control_plane else [], mcp_stdio_enabled=is_mcp_stdio_enabled(), use_si_terminology=use_si_terminology, + claude_code_marketplace_auth_required=marketplace_auth_required(), ) diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 62c973196b9..d689ab2a537 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -2,7 +2,10 @@ import asyncio import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final + +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -19,6 +22,17 @@ if TYPE_CHECKING: from litellm.router import Router +@with_config(ConfigDict(extra="allow", strict=True)) +class _CachedHealthCheckResults(TypedDict): + healthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + unhealthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + timestamp: NotRequired[ReadOnly[int | float]] + checked_by: NotRequired[ReadOnly[object]] + + +_CACHED_HEALTH_CHECK_RESULTS: Final = TypeAdapter(_CachedHealthCheckResults) + + class SharedHealthCheckManager: """ Manager for coordinating health checks across multiple pods using Redis. @@ -109,7 +123,7 @@ class SharedHealthCheckManager: verbose_proxy_logger.error("Error releasing health check lock: %s", str(e)) @with_service_target(HEALTH_CHECKS_TARGET) - async def get_cached_health_check_results(self) -> dict[str, Any] | None: + async def get_cached_health_check_results(self) -> _CachedHealthCheckResults | None: """ Get cached health check results from Redis. @@ -127,10 +141,9 @@ class SharedHealthCheckManager: return None # Parse the cached data - if isinstance(cached_data, str): - cached_results = json.loads(cached_data) - else: - cached_results = cached_data + cached_results: Final = _CACHED_HEALTH_CHECK_RESULTS.validate_python( + json.loads(cached_data) if isinstance(cached_data, str) else cached_data + ) # Check if the cache is still valid cache_timestamp: Final = cached_results.get("timestamp", 0) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index e44d9304ac1..a3a6f582dce 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.proxy.spend_tracking.input_tokens import tokenizer_models from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: return resolve_model_group_alias(llm_router.model_group_alias, model) +def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]: + from litellm.proxy.proxy_server import llm_router + + return tokenizer_models(model, llm_router) + + def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: from litellm.proxy.proxy_server import llm_router @@ -884,6 +891,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ] = _resolve_ptu_team_ceiling_via_proxy_router, fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings, force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings, + tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now @@ -893,6 +901,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver + self._tokenizer_models_resolver = tokenizer_models_resolver if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( BATCH_RATE_LIMITER_SCRIPT @@ -4009,15 +4018,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: estimate: Final = max( 0, - int( - token_counter( - model=model or "", - messages=countable_messages, - text=selected_text, - tools=countable_tools, - tool_choice=countable_tool_choice, - use_default_image_token_count=True, + *( + int( + token_counter( + model=tokenizer_model, + messages=countable_messages, + text=selected_text, + tools=countable_tools, + tool_choice=countable_tool_choice, + use_default_image_token_count=True, + ) ) + for tokenizer_model in self._tokenizer_models_resolver(model or "") ), ) return estimate + audio_token_estimate diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 27b5924c540..771d68a0ea0 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai" -_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"}) +_RESPONSES_API_ROUTES: Final = frozenset({"/v1/responses", "/responses"}) _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE: Final[Mapping[str, str]] = MappingProxyType( { "aresponses": "previous_response_id", @@ -104,7 +104,7 @@ def _rewrite_advertised_id( return event -def _is_responses_api_create_route(request_route: str | None) -> bool: +def _streams_response_ids(request_route: str | None) -> bool: if request_route is None: return False canonical: Final = ( @@ -112,7 +112,8 @@ def _is_responses_api_create_route(request_route: str | None) -> bool: if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/") else request_route ) - return canonical in _RESPONSES_API_CREATE_ROUTES + parent, _, response_id = canonical.rpartition("/") + return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "") class ResponsesIDSecurity(CustomLogger): @@ -339,7 +340,7 @@ class ResponsesIDSecurity(CustomLogger): async for chunk in response: if ( isinstance(chunk, BaseLiteLLMOpenAIResponseObject) - and _is_responses_api_create_route(user_api_key_dict.request_route) + and _streams_response_ids(user_api_key_dict.request_route) and not general_settings.get("disable_responses_id_security", False) ): chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 22c4f27d94a..a980b9c33ff 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -16,6 +16,7 @@ import json import os import re import secrets +from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from html import escape @@ -42,7 +43,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator +from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator import litellm from litellm._internal_context import with_service_target @@ -106,6 +107,7 @@ from litellm.proxy.common_utils.html_forms.default_credentials_hint import shoul from litellm.proxy.common_utils.html_forms.jwt_display_template import ( jwt_display_template, ) +from litellm.proxy.common_utils.html_forms.native_client_consent import team_select_field from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -184,6 +186,8 @@ _CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS: Final = 60 _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: Final = 30 _CLI_SSO_USER_CODE_ALPHABET: Final = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" _CLI_SSO_LOGIN_ID_RE: Final = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$") +CLI_SSO_BROWSER_TEAM_CHOICE: Final = "browser_team_choice" +CLI_SSO_CHOSEN_TEAM_ID: Final = "chosen_team_id" _CLI_SSO_USER_CODE_RE = re.compile(rf"^[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}-[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}$") _CLI_SSO_SCALAR_TYPES: Final = (str, int, float, bool) _CLI_SSO_DEST_KEY_RE: Final = re.compile(r"^[A-Za-z0-9_.-]+$") @@ -670,6 +674,7 @@ def _render_cli_sso_verification_page( verify_url: str, browser_complete_token: str, prefill_user_code: str | None = None, + team_choices: Sequence[tuple[str, str]] = (), ) -> str: escaped_verify_url: Final = escape(verify_url, quote=True) escaped_browser_complete_token: Final = escape(browser_complete_token, quote=True) @@ -716,6 +721,15 @@ def _render_cli_sso_verification_page( letter-spacing: 0.08em; text-transform: uppercase; }} + select {{ + box-sizing: border-box; + width: 100%; + padding: 12px; + margin-bottom: 18px; + border: 1px solid #cbd5e1; + border-radius: 6px; + font-size: 16px; + }} button {{ margin-top: 16px; width: 100%; @@ -735,6 +749,7 @@ def _render_cli_sso_verification_page(

{instructions}

+ {team_select_field(team_choices, require_choice=True)} @@ -823,13 +838,46 @@ async def cli_sso_complete(request: Request, login_id: str): ): raise HTTPException(status_code=400, detail="Invalid verification code") - flow["user_code_verified"] = True - _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow) + team_choice_flow: Final = _CliSsoTeamChoiceFlow.model_validate(flow) + chosen_team_id: Final = _browser_chosen_team_id(team_choice_flow, form_values) + if chosen_team_id is not None and chosen_team_id not in team_choice_flow.teams: + raise HTTPException(status_code=400, detail="Choose one of the teams you belong to") + + _set_cli_sso_flow( + login_id=login_id, + cache=cli_sso_session_cache, + flow={ + **flow, + "user_code_verified": True, + CLI_SSO_CHOSEN_TEAM_ID: chosen_team_id, + "session_data": ( + flow["session_data"] + if chosen_team_id is None + else { + **flow["session_data"], + "teams": [chosen_team_id, *(team for team in team_choice_flow.teams if team != chosen_team_id)], + } + ), + }, + ) html_content: Final = render_cli_sso_success_page() return HTMLResponse(content=html_content, status_code=200) +class _CliSsoTeamChoiceFlow(BaseModel): + model_config = ConfigDict(frozen=True) + + browser_team_choice: bool = Field(default=False, validation_alias=CLI_SSO_BROWSER_TEAM_CHOICE) + teams: tuple[str, ...] = Field(default=(), validation_alias=AliasPath("session_data", "teams")) + + +def _browser_chosen_team_id(flow: _CliSsoTeamChoiceFlow, form_values: Mapping[str, Sequence[str]]) -> str | None: + if not flow.browser_team_choice: + return None + return (form_values.get("team_id") or [""])[0] or None + + def normalize_email(email: str | None) -> str | None: """ Normalize email address to lowercase for consistent storage and comparison. @@ -2416,6 +2464,18 @@ def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str return [detail.team_id for detail in team_details if detail.team_id is not None] +def _browser_team_choices( + flow: _CliSsoTeamChoiceFlow, team_details: Sequence[CliSsoTeamDetail] +) -> tuple[tuple[str, str], ...]: + if not flow.browser_team_choice: + return () + labeled: Final = tuple( + (detail.team_id, detail.team_alias or detail.team_id) for detail in team_details if detail.team_id is not None + ) + label_counts: Final = Counter(label for _, label in labeled) + return tuple((team_id, label if label_counts[label] == 1 else f"{label} ({team_id})") for team_id, label in labeled) + + def selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> CliSsoTeamDetail | None: """``None`` means the team's grants are unknown. An empty grant is a real value meaning unrestricted, so an unknown one must not be minted as empty.""" @@ -2484,6 +2544,7 @@ async def _complete_cli_sso_callback_session( request.scope.get("litellm_microsoft_interactive_subject"), resolved_user_id, prisma_client ) resolved_teams: Final = _cli_sso_session_teams(team_details) + team_choices: Final = _browser_team_choices(_CliSsoTeamChoiceFlow.model_validate(flow), team_details) attribution_metadata: Final = build_cli_sso_attribution_metadata(result=result) if attribution_metadata: await _persist_cli_sso_user_metadata( @@ -2521,6 +2582,7 @@ async def _complete_cli_sso_callback_session( verify_url=verify_url, browser_complete_token=browser_complete_token, prefill_user_code=prefill_user_code, + team_choices=team_choices, ), status_code=200, ) diff --git a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py index c417187396f..6c6be219a92 100644 --- a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py +++ b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py @@ -43,6 +43,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span from litellm.proxy.mcp_server.oauth_utils import canonicalize_url_identity from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import MCPServerRepository # The actor the removed discovery write-back stamped rows with. _DISCOVERY_ACTOR: Final = "mcp_oauth_discovery" @@ -106,7 +107,7 @@ def _is_stamped_issuer_row(row: _MCPServerRow) -> bool: async def backfill_discovery_stamped_issuers(prisma_client: PrismaClient) -> int: """Clear gateway-written issuer stamps, returning the number of rows healed.""" - candidate_rows: Final[list[_MCPServerRow]] = await prisma_client.db.litellm_mcpservertable.find_many( + candidate_rows: Final = await MCPServerRepository(prisma_client).table.find_many( where={ "updated_by": _DISCOVERY_ACTOR, "auth_type": {"in": list(_AUTH_TYPES_WITH_ISSUER_ANCHORING)}, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 972ec18d487..f96fab75bd1 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -68,6 +68,7 @@ from litellm.llms.deepgram.common_utils import ( from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.openai.common_utils import OpenAIError as LiteLLMOpenAIError +from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_api_resource from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -3530,6 +3531,10 @@ async def deepgram_listen_websocket_route( ) +_OPENAI_BETA_HEADER: Final = "openai-beta" +_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"}) + + class BaseOpenAIPassThroughHandler: @staticmethod async def base_openai_pass_through_handler( @@ -3540,7 +3545,7 @@ class BaseOpenAIPassThroughHandler: base_target_url: str, api_key: str | None, custom_llm_provider: litellm.LlmProviders, - extra_headers: dict | None = None, + extra_headers: Mapping[str, str] | None = None, ): encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -3565,7 +3570,11 @@ class BaseOpenAIPassThroughHandler: endpoint=endpoint, target=str(updated_url), custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( - api_key=api_key, request=request, extra_headers=extra_headers + api_key=api_key, + endpoint=endpoint, + request=request, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, ), is_streaming_request=is_streaming_request, custom_llm_provider=( @@ -3587,27 +3596,41 @@ class BaseOpenAIPassThroughHandler: _base_openai_pass_through_handler = base_openai_pass_through_handler @staticmethod - def _append_openai_beta_header(headers: dict, request: Request) -> dict: - """ - Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request - """ - if RouteChecks.is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: - headers["OpenAI-Beta"] = "assistants=v2" - return headers + def _openai_beta(endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders) -> str | None: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and openai_api_resource(endpoint) in _CLIENT_OPENAI_BETA_RESOURCES + ): + return request.headers.get(_OPENAI_BETA_HEADER) or ( + "agents=v1" if is_openai_agents_api_path(endpoint) else None + ) + return "assistants=v2" if RouteChecks.is_assistants_api_request(request) else None @staticmethod - def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict: - base_headers = {} - if api_key is not None: - base_headers = { - "authorization": f"Bearer {api_key}", - "api-key": f"{api_key}", - } - if extra_headers is not None: - base_headers.update(extra_headers) + def _append_openai_beta_header( + headers: Mapping[str, str], endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders + ) -> dict[str, str]: + if any(name.lower() == _OPENAI_BETA_HEADER for name in headers): + return {**headers} + openai_beta: Final = BaseOpenAIPassThroughHandler._openai_beta(endpoint, request, custom_llm_provider) + return {**headers} if openai_beta is None else {**headers, _OPENAI_BETA_HEADER: openai_beta} + + @staticmethod + def _assemble_headers( + api_key: str | None, + endpoint: str, + request: Request, + custom_llm_provider: litellm.LlmProviders, + extra_headers: Mapping[str, str] | None = None, + ) -> dict[str, str]: + api_key_headers: Final[Mapping[str, str]] = ( + {} if api_key is None else {"authorization": f"Bearer {api_key}", "api-key": api_key} + ) return BaseOpenAIPassThroughHandler._append_openai_beta_header( - headers=base_headers, + headers={**api_key_headers, **(extra_headers or {})}, + endpoint=endpoint, request=request, + custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a11b0be3983..4eeb9b26174 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,10 +1,12 @@ from datetime import datetime from typing import Final +from urllib.parse import urlparse import httpx import litellm from litellm import stream_chunk_builder +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -16,7 +18,7 @@ from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig, CohereEmbeddingRowShape from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, @@ -29,6 +31,12 @@ from litellm.types.utils import ( from .base_passthrough_logging_handler import BasePassthroughLoggingHandler +COHERE_EMBED_PATHS: Final = ("/v1/embed", "/v2/embed") + + +def is_cohere_embed_route(url_route: str) -> bool: + return urlparse(url_route).path.rstrip("/").endswith(COHERE_EMBED_PATHS) + class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): @property @@ -82,28 +90,24 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle Cohere passthrough logging with route detection and cost tracking. """ - # Check if this is an embed endpoint - if "/v1/embed" in url_route and "/v1/embeddings" not in url_route: + if is_cohere_embed_route(url_route): model: Final = request_body.get("model", response_body.get("model", "")) try: - cohere_embed_config: Final = CohereEmbeddingConfig() - litellm_model_response = litellm.EmbeddingResponse() handler_instance: Final = CoherePassthroughLoggingHandler() input_texts = request_body.get("texts", []) if not input_texts: input_texts = request_body.get("input", []) - # Transform the response - litellm_model_response = cohere_embed_config.transform_response( + litellm_model_response: Final = CohereEmbeddingConfig().transform_native_embed_response( response=httpx_response, api_key="", logging_obj=logging_obj, data=request_body, - model_response=litellm_model_response, model=model, encoding=litellm.encoding, input=input_texts, + row_shape=CohereEmbeddingRowShape.TYPED, ) # Calculate cost using LiteLLM's cost calculator @@ -153,20 +157,11 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): "result": litellm_model_response, "kwargs": kwargs, } - except Exception: - # For other routes (e.g., /v2/chat), fall back to chat handler - return super().passthrough_chat_handler( - httpx_response=httpx_response, - response_body=response_body, - logging_obj=logging_obj, - url_route=url_route, - result=result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - request_body=request_body, - **kwargs, + except Exception as e: + verbose_proxy_logger.warning( + "Could not price the Cohere passthrough embed response, logging it without a cost: %s", e ) + return {"result": None, "kwargs": kwargs} # For non-embed routes (e.g., /v2/chat), fall back to chat handler return super().passthrough_chat_handler( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c410150df70..b98dd42e3de 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,7 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.llms.openai.common_utils import is_openai_agents_api_path from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -38,7 +40,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, PassthroughCallTypes +from litellm.types.utils import ( + EmbeddingResponse, + ImageResponse, + LlmProviders, + PassthroughCallTypes, + StandardPassThroughResponseObject, +) from litellm.utils import ModelResponse, TextCompletionResponse, convert_to_model_response_object # Hostnames that route to OpenAI-compatible APIs. @@ -121,6 +129,52 @@ def _responses_sub_route(path: str) -> tuple[str, ...] | None: return None if anchor is None else tuple(segments[anchor + 1 :]) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +class _AgentConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + model: str + + +class _AgentsApiBody(BaseModel): + model_config = ConfigDict(frozen=True) + + agent: _AgentConfig + + +class _AgentsApiSessionEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + session: _AgentsApiBody + + +_AGENTS_API_BODY: Final = TypeAdapter(_AgentsApiBody | _AgentConfig) + + +def _agent_model(body: Mapping[str, object]) -> str | None: + try: + parsed: Final = _AGENTS_API_BODY.validate_python(body) + except ValidationError: + return None + return parsed.agent.model if isinstance(parsed, _AgentsApiBody) else parsed.model + + +def _session_event_model(chunk: str) -> str | None: + event: Final = BaseModelResponseIterator.string_to_dict_parser(str_line=chunk) + if event is None: + return None + try: + return _AgentsApiSessionEvent.model_validate(event).session.agent.model + except ValidationError: + return None + + +def _streamed_agent_model(all_chunks: Sequence[str]) -> str | None: + return next((model for chunk in all_chunks if (model := _session_event_model(chunk)) is not None), None) + + def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(part, Mapping) or part.get("type") != "image_url": return False @@ -220,14 +274,44 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return "billed" @staticmethod - def _label_unbilled_responses_call( + def is_openai_agents_route(url_route: str) -> bool: + return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) + + @staticmethod + def _unbilled_call_model( + url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object] + ) -> str | None: + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return _agent_model(response_body) or _agent_model(request_body) + response_model: Final = response_body.get("model") + if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(response_model, str): + return response_model + return None + + @staticmethod + def _label_model(logging_obj: LiteLLMLoggingObj, model: str | None) -> None: + if model is not None: + logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + + @staticmethod + def _label_unbilled_call( logging_obj: LiteLLMLoggingObj, url_route: str, + request_body: Mapping[str, object], response_body: Mapping[str, object], ) -> None: - model: Final = response_body.get("model") - if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str): - logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + OpenAIPassthroughLoggingHandler._label_model( + logging_obj, OpenAIPassthroughLoggingHandler._unbilled_call_model(url_route, request_body, response_body) + ) + + @staticmethod + def _log_agents_stream( + litellm_logging_obj: LiteLLMLoggingObj, request_body: Mapping[str, object], all_chunks: Sequence[str] + ) -> PassThroughEndpointLoggingTypedDict: + OpenAIPassthroughLoggingHandler._label_model( + litellm_logging_obj, _streamed_agent_model(all_chunks) or _agent_model(request_body) + ) + return {"result": StandardPassThroughResponseObject(response="\n".join(all_chunks)), "kwargs": {}} @staticmethod def is_openai_embeddings_route(url_route: str) -> bool: @@ -397,7 +481,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_responses: Final = responses_billing != "unbilled" if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): - OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body) + OpenAIPassthroughLoggingHandler._label_unbilled_call( + logging_obj, + url_route, + request_body=_STR_MAPPING.validate_python(request_body), + response_body=_STR_MAPPING.validate_python(response_body), + ) return { "result": None, "kwargs": kwargs, @@ -676,6 +765,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle logging for collected OpenAI streaming chunks with cost tracking. """ + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return OpenAIPassthroughLoggingHandler._log_agents_stream( + litellm_logging_obj, _STR_MAPPING.validate_python(request_body), all_chunks + ) try: is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) terminal_response: Final = ( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..27e31c8f74d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -22,6 +22,7 @@ from .llm_provider_handlers.assembly_passthrough_logging_handler import ( ) from .llm_provider_handlers.cohere_passthrough_logging_handler import ( CoherePassthroughLoggingHandler, + is_cohere_embed_route, ) from .llm_provider_handlers.cursor_passthrough_logging_handler import ( CursorPassthroughLoggingHandler, @@ -91,7 +92,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages", "/v1/messages/batches"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat"] self.assemblyai_passthrough_logging_handler = AssemblyAIPassthroughLoggingHandler() # Langfuse @@ -536,13 +537,7 @@ class PassThroughEndpointLogging: return False def is_cohere_route(self, url_route: str) -> bool: - for route in self.TRACKED_COHERE_ROUTES: - if route not in url_route: - continue - if route == "/v1/embed" and "/v1/embeddings" in url_route: - continue - return True - return False + return is_cohere_embed_route(url_route) or any(route in url_route for route in self.TRACKED_COHERE_ROUTES) def is_assemblyai_route(self, url_route: str): parsed_url: Final = urlparse(url_route) @@ -645,6 +640,7 @@ class PassThroughEndpointLogging: or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) ) def _set_cost_per_request( diff --git a/litellm/proxy/plugin_routes.py b/litellm/proxy/plugin_routes.py index 1dfe5cbc6bb..cfbcfefe9e8 100644 --- a/litellm/proxy/plugin_routes.py +++ b/litellm/proxy/plugin_routes.py @@ -31,6 +31,8 @@ from typing import Final from cryptography.fernet import Fernet, InvalidToken from fastapi import APIRouter, Depends, HTTPException, Request, Response +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import PluginConfig, SpecialHeaders, UserAPIKeyAuth @@ -39,6 +41,18 @@ from litellm.types.llms.custom_http import httpxSpecialProvider router: Final = APIRouter() + +@with_config(ConfigDict(extra="allow", strict=True)) +class PluginSessionClaim(TypedDict): + plugin: NotRequired[ReadOnly[object]] + user_id: NotRequired[ReadOnly[str]] + user_role: NotRequired[ReadOnly[str]] + exp: NotRequired[ReadOnly[int | float | str]] + + +_PLUGIN_SESSION_CLAIM: Final = TypeAdapter(PluginSessionClaim) + + # Hop-by-hop headers (RFC 7230) and the litellm session cookie — never forwarded # to a plugin backend. Credential headers are added on top per-request from the # canonical SpecialHeaders set so the plugin only ever authenticates via its own @@ -155,7 +169,7 @@ def issue_plugin_session_claim(plugin_name: str, user_id: str | None, user_role: return _plugin_fernet(plugin_name).encrypt(json.dumps(claim).encode()).decode() -def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: +def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> PluginSessionClaim: """Verify and decode a plugin session claim. Raises ValueError if the HMAC is invalid, the audience is wrong, or @@ -163,7 +177,7 @@ def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: """ try: raw: Final = _plugin_fernet(plugin_name).decrypt(ciphertext.encode(), ttl=_CLAIM_TTL_SECONDS) - claim: Final = json.loads(raw) + claim: Final = _PLUGIN_SESSION_CLAIM.validate_python(json.loads(raw)) except (InvalidToken, Exception) as exc: raise ValueError("Invalid, tampered, or expired plugin session claim") from exc diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index 59d15b0ba0a..2d0e9d25368 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -11,7 +11,7 @@ from collections.abc import Mapping, Sequence from itertools import chain from typing import TYPE_CHECKING, Any, Final, Optional -from litellm._logging import print_startup_message, verbose_proxy_logger +from litellm._logging import print_startup_message, print_startup_warning, verbose_proxy_logger from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_validator import PolicyValidator @@ -93,7 +93,7 @@ def _print_policies_on_startup( ) else: print_startup_message("\n".join((header, *policy_lines))) - print_startup_message( + print_startup_warning( f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}\n", ) sys.stdout.flush() diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 909f6de40cd..0d8ab98a0b1 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -10,16 +10,17 @@ import sys import urllib.parse as urlparse from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Annotated, Any, Final import click import httpx from click.core import ParameterSource from dotenv import load_dotenv -from pydantic import ConfigDict +from pydantic import BeforeValidator, ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm -from litellm._logging import print_startup_message +from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, @@ -205,7 +206,7 @@ def deprecated_v2_flag_passed_on_cli() -> bool: def _print_enforce_prisma_migration_check_notice() -> None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " "deprecated and has no effect, because the proxy always exits " "when database setup fails at startup. You can safely remove it.\033[0m", @@ -227,6 +228,36 @@ def _print_or_hold_enforce_prisma_migration_check_notice( return print_once +def _empty_as_none(value: object) -> object: + return value or None + + +class _ConfigStartupOutputSettings(TypedDict, total=False): + litellm_settings: ReadOnly[Annotated[StartupOutputSettings | None, BeforeValidator(_empty_as_none)]] + + +_CONFIG_STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(_ConfigStartupOutputSettings) + + +def _apply_config_suppress_startup_output(litellm_settings: StartupOutputSettings | None) -> None: + if litellm_settings is None or litellm_settings.get("suppress_startup_output") is not True: + return + litellm.suppress_startup_output = True + os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true" + + +def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | None) -> None: + """Turn on the config's json_logs and suppress_startup_output before any worker starts. Workers that + uvicorn or granian spawn import litellm afresh and read both from the env at import, so they log JSON + and stay quiet from their first line instead of from when load_config reaches the key.""" + _apply_config_suppress_startup_output(litellm_settings) + if litellm_settings is None or litellm_settings.get("json_logs") is not True: + return + litellm.json_logs = True + os.environ["JSON_LOGS"] = "true" + litellm.turn_on_json() + + class ProxyInitializationHelpers: @staticmethod def echo_litellm_version(): @@ -373,7 +404,7 @@ class ProxyInitializationHelpers: if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -396,7 +427,7 @@ class ProxyInitializationHelpers: import uvicorn if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -404,7 +435,7 @@ class ProxyInitializationHelpers: if "limit_max_requests_jitter" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["limit_max_requests_jitter"] = jitter else: - print_startup_message( + print_startup_warning( f"\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " f"requires uvicorn>=0.41.0, but installed uvicorn=={uvicorn.__version__}. " f"Ignoring the flag.\033[0m", @@ -571,12 +602,12 @@ class ProxyInitializationHelpers: print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n") if max_requests_before_restart is not None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM: --max_requests_before_restart is not supported by Granian " "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n", ) if ciphers is not None: - print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") + print_startup_warning("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") kwargs: Final[dict[str, object]] = { "target": "litellm.proxy.proxy_server:app", @@ -698,7 +729,7 @@ class ProxyInitializationHelpers: gunicorn_options["max_requests"] = max_requests_before_restart if max_requests_before_restart_jitter is not None: if max_requests_before_restart is None: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " "has no effect without --max_requests_before_restart\033[0m\n", ) @@ -739,7 +770,7 @@ class ProxyInitializationHelpers: with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - print_startup_message(f""" + print_startup_warning(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` """) @@ -1325,6 +1356,19 @@ def run_server( os.environ[k] = v litellm_settings = None + env_config_file: Final = os.environ.get("CONFIG_FILE_PATH") + if ( + env_config_file + and os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is None + and ProxyConfig().is_yaml(config_file_path=env_config_file) + ): + import asyncio + + _apply_config_suppress_startup_output( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python( + asyncio.run(ProxyConfig().read_config_file(config_file_path=env_config_file)) + ).get("litellm_settings") + ) if config is not None: """ Allow user to pass in db url via config @@ -1342,17 +1386,9 @@ def run_server( ### LITELLM SETTINGS ### litellm_settings = _config.get("litellm_settings", None) - if ( - litellm_settings is not None - and "json_logs" in litellm_settings - and litellm_settings["json_logs"] is True - ): - import litellm - - litellm.json_logs = True - os.environ["JSON_LOGS"] = "true" - - litellm.turn_on_json() + _apply_config_logging_settings( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python(_config).get("litellm_settings") + ) if held_prisma_notice is not None: held_prisma_notice() ### GENERAL SETTINGS ### @@ -1450,7 +1486,9 @@ def run_server( _bad_scheme = unsupported_db_scheme(_candidate_url) if _bad_scheme is not None: print( - f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1553,7 +1591,7 @@ def run_server( env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"), ) if deprecated_v2_flag_passed_on_cli() and use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " "deprecated and has no effect, because the v2 migration " "resolver is now the default. You can safely remove it. To " @@ -1561,7 +1599,7 @@ def run_server( "--use_legacy_migration_resolver.\033[0m", ) if not use_v2_resolver: - print_startup_message( + print_startup_warning( "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration " "resolver. It performs the diff-and-force recovery that can " "cause schema thrashing during rolling deploys where two " @@ -1583,22 +1621,26 @@ def run_server( # issues, and any `prisma db push` against a # partitioned LiteLLM_SpendLogs. print( - f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", + terminal_text(f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m"), file=sys.stderr, flush=True, ) sys.exit(2) if not setup_ok: print( - "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " - "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " + "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + ) ) sys.exit(1) else: print( - "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " - "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " - "or run a shipped LiteLLM image.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " + "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " + "or run a shipped LiteLLM image.\033[0m" + ) ) sys.exit(1) pgbouncer_settings: Final = PgBouncerSettings() @@ -1609,8 +1651,10 @@ def run_server( ) if isinstance(pooled_database_url, PgBouncerError): print( - f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " - f"could not start: {pooled_database_url.reason}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1675,7 +1719,7 @@ def run_server( run_gunicorn=run_gunicorn is True, ) if single_worker_warning is not None: - print_startup_message(f"\033[1;33m{single_worker_warning}\033[0m\n") + print_startup_warning(f"\033[1;33m{single_worker_warning}\033[0m\n") if run_gunicorn is False and run_hypercorn is False and run_granian is False: if limit_concurrency is not None: uvicorn_args["limit_concurrency"] = limit_concurrency diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bb8fffd85bc..091e05610ba 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -100,7 +100,11 @@ from litellm.litellm_core_utils.litellm_logging import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, offload_token_count +from litellm.litellm_core_utils.token_counter import ( + UnsupportedContentTypeError, + deployment_openai_model, + offload_token_count, +) from litellm.llms.bedrock.count_tokens.kimi_k3 import ( HUGGING_FACE_SOURCE, KimiK3Encodings, @@ -306,6 +310,7 @@ import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target from litellm._logging import ( + STARTUP_OUTPUT_SETTINGS, print_startup_message, redact_string, verbose_proxy_logger, @@ -5801,6 +5806,12 @@ class ProxyConfig: # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") return config + async def read_config_file(self, config_file_path: str) -> Mapping[str, object]: + """The config file with its includes merged, without the bucket, database and secret steps of get_config.""" + return await self._process_includes( + config=self._load_yaml_file(config_file_path), config_file_path=os.path.abspath(config_file_path) + ) + async def _process_includes(self, config: dict, config_file_path: str) -> dict: """ Process includes by appending their contents to the main config @@ -6425,13 +6436,8 @@ class ProxyConfig: search_tools_parsed: Final[list[SearchToolTypedDict]] = [] - print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") - for search_tool in search_tools_raw: - # Display loaded search tool search_tool_name = search_tool.get("search_tool_name", "") - search_provider = search_tool.get("litellm_params", {}).get("search_provider", "") - print_startup_message(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # Handle os.environ/ variables in litellm_params litellm_params = search_tool.get("litellm_params", {}) @@ -6601,13 +6607,15 @@ class ProxyConfig: from litellm.types.integrations.prometheus import validate_caller_identity_settings validate_caller_identity_settings(litellm_settings) + if STARTUP_OUTPUT_SETTINGS.validate_python(litellm_settings).get("suppress_startup_output") is True: + litellm.suppress_startup_output = True # ANSI escape code for blue text blue_color_code: Final = "\033[94m" reset_color_code: Final = "\033[0m" for key, value in litellm_settings.items(): if key == "cache" and value is True: - print_startup_message(f"{blue_color_code}\nSetting Cache on Proxy{reset_color_code}") + print_startup_message(f"\n{blue_color_code}Setting Cache on Proxy{reset_color_code}") from litellm.caching.caching import Cache cache_params = {} @@ -6922,6 +6930,8 @@ class ProxyConfig: setattr(litellm, key, value) elif key == "drop_params": litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger) + elif key == "suppress_startup_output": + continue else: verbose_proxy_logger.debug( "%s setting litellm.%s=%s%s", @@ -6968,6 +6978,7 @@ class ProxyConfig: for name in ( "vector_store_deny_by_default", "search_tool_deny_by_default", + "claude_code_marketplace_auth_required", "vertex_ai_pass_through_drop_anthropic_betas", "request_tag_mappings", ) @@ -7277,6 +7288,13 @@ class ProxyConfig: ## SEARCH TOOLS SETTINGS search_tools: Final[list[SearchToolTypedDict] | None] = self.parse_search_tools(config) + if search_tools: + print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") + for tool in search_tools: + print_startup_message( + f"\033[32m {tool.get('search_tool_name', '')} " + f"({tool.get('litellm_params', {}).get('search_provider', '')})\033[0m" + ) ## SANDBOX TOOLS SETTINGS from litellm.sandbox.sandbox_tools import register_sandbox_tools @@ -9679,12 +9697,9 @@ async def initialize( master_key, \ user_custom_auth, \ prisma_client - from litellm.proxy.common_utils.banner import LITELLM_BANNER, show_banner + from litellm.proxy.common_utils.banner import LITELLM_BANNER - if litellm.json_logs: - print_startup_message(LITELLM_BANNER) - else: - show_banner() + print_startup_message(f"\n{LITELLM_BANNER}\n") if os.getenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "").lower() != "true": generate_feedback_box() user_model = model @@ -15542,7 +15557,7 @@ async def _count_request_tokens( count_source="local_estimate", ) estimate: Final = await _local_token_estimate( - model=model_to_use, + model=deployment_openai_model(deployment_view) or model_to_use, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 2e72cf3d984..4237517508b 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2023,6 +2023,34 @@ ], "default_model_placeholder": "jina_ai/" }, + { + "provider": "KYMA", + "provider_display_name": "Kyma API", + "litellm_provider": "kyma", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://kymaapi.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "kyma/qwen3.7-flash" + }, { "provider": "Laya", "provider_display_name": "Laya", @@ -4228,5 +4256,33 @@ } ], "default_model_placeholder": "cursor/claude-4-sonnet" + }, + { + "provider": "BIGMODEL", + "provider_display_name": "BigModel (Zhipu AI)", + "litellm_provider": "bigmodel", + "credential_fields": [ + { + "key": "api_key", + "label": "BigModel (Zhipu AI) API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://open.bigmodel.cn", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": "https://open.bigmodel.cn" + } + ], + "default_model_placeholder": "bigmodel/glm-5.3" } ] diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 9f33e0d2adc..b8cdea0dea4 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -367,19 +367,23 @@ async def get_mcp_servers(): tags=["public", "Claude Code Marketplace"], ) async def public_skill_hub(): - """Return enabled (public) Claude Code skills — no auth required.""" + """Return enabled (public) Claude Code skills, or none when the marketplace requires a key.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( MANIFEST_FIELDS, get_installation_preference, - get_prisma_client, ) + from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required + from litellm.proxy.proxy_server import prisma_client from litellm.types.proxy.claude_code_endpoints import ( ListPluginsResponse, PluginListItem, ) + if marketplace_auth_required(): + return ListPluginsResponse(plugins=[], count=0) + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - prisma_client: Final = await get_prisma_client() plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) items: Final = [] for plugin in plugins: @@ -406,7 +410,8 @@ async def public_skill_hub(): ) return ListPluginsResponse(plugins=items, count=len(items)) except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + verbose_logger.exception("Error listing public skills: %s", e) + raise HTTPException(status_code=500, detail="Failed to list skills") @router.get( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 4f8b1697f7e..9e642a02d49 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.litellm_core_utils.token_counter import usage_tokenizer_model from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, @@ -85,6 +86,7 @@ from litellm.proxy.route_llm_request import raise_if_required_body_param_missing from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD +from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel @@ -96,6 +98,7 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import DeleteResponseResult from litellm.types.token_count import TokenCountSourceFields from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse +from litellm.utils import get_base_model_from_metadata if TYPE_CHECKING: from litellm.proxy.utils import ProxyLogging @@ -104,11 +107,43 @@ router: Final = APIRouter() _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None) _DECODED_JSON: Final = TypeAdapter(object) _PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_RETRIEVE_QUERY_PARAMS: Final = tuple(sorted(RETRIEVE_INCLUDE_QUERY_KEYS | RETRIEVE_STREAM_QUERY_KEYS)) +_RETRIEVE_QUERY_OPENAPI: Final = { + "parameters": [ + { + "name": "stream", + "in": "query", + "required": False, + "schema": {"type": "boolean"}, + "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)", + }, + { + "name": "starting_after", + "in": "query", + "required": False, + "schema": {"type": "integer", "minimum": 0}, + "description": "With stream=true, replay only the events after this sequence number", + }, + { + "name": "include", + "in": "query", + "required": False, + "schema": {"type": "array", "items": {"type": "string"}}, + "description": "Additional output data to include, forwarded to the provider", + }, + { + "name": "include_obfuscation", + "in": "query", + "required": False, + "schema": {"type": "boolean"}, + "description": "With stream=true, whether delta events carry obfuscation padding", + }, + ] +} _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg -RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}} -RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { +RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { 200: { "model": ResponsesAPIResponse, "content": { @@ -252,19 +287,19 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None: "/v1/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) @router.post( "/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) @router.post( "/openai/v1/responses", dependencies=[Depends(user_api_key_auth)], tags=["responses"], - responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, + responses=RESPONSES_API_RESPONSE_SCHEMAS, ) async def responses_api( request: Request, @@ -537,7 +572,10 @@ async def cursor_model_list( def _dispatched_prompt_token_counter(logging_obj: LiteLLMLoggingObj) -> Callable[[], int]: return lambda: count_request_input_tokens( - model=str(logging_obj.model_call_details.get("model") or ""), + model=usage_tokenizer_model( + str(logging_obj.model_call_details.get("model") or ""), + get_base_model_from_metadata(logging_obj.model_call_details), + ), request_input=messages_since_newest_compaction(logging_obj.model_call_details.get("messages")), system_input=None, ) @@ -744,18 +782,21 @@ async def cursor_chat_completions( dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) @router.get( "/responses/{response_id}", dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) @router.get( "/openai/v1/responses/{response_id}", dependencies=[Depends(user_api_key_auth)], tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, + openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) async def get_response( response_id: str, @@ -780,6 +821,10 @@ async def get_response( # Get provider response curl -X GET http://localhost:4000/v1/responses/resp_abc123 \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" + + # Resume a background response's event stream after sequence number 7 + curl -N "http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7" \ + -H "Authorization: Bearer $LITELLM_MASTER_KEY" ``` """ from litellm.proxy.proxy_server import ( @@ -827,6 +872,14 @@ async def get_response( data: Final = await read_request_body(request=request) read_settles_billing: Final = not carries_client_side_overrides(data) data["response_id"] = response_id + retrieve_query: Final = { + key: values[0] if len(values) == 1 else values + for key in _RETRIEVE_QUERY_PARAMS + if (values := request.query_params.getlist(key)) + } + if retrieve_query: + body_extra_query: Final[object] = data.get("extra_query") + data["extra_query"] = {**(body_extra_query if isinstance(body_extra_query, dict) else {}), **retrieve_query} processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: response: Final[object] = await processor.base_process_llm_request( @@ -838,7 +891,12 @@ async def get_response( llm_router=llm_router, general_settings=general_settings, proxy_config=proxy_config, - select_data_generator=select_data_generator, + select_data_generator=partial( + _streamed_retrieval_data_generator, + partial(select_data_generator, responses_stream_errors=True), + response_id, + read_settles_billing, + ), model=None, user_model=user_model, user_temperature=user_temperature, @@ -912,12 +970,24 @@ async def _bill_finished_background_response( fastapi_response: Response, read_settles_billing: bool, ) -> None: + _report_charge( + await _settle_finished_background_read(response, response_id, user_api_key_dict, read_settles_billing), + fastapi_response, + ) + + +async def _settle_finished_background_read( + response: object, + response_id: str, + user_api_key_dict: UserAPIKeyAuth, + read_settles_billing: bool, +) -> "_BilledCharge | None": from litellm.proxy.proxy_server import proxy_logging_obj if not isinstance(response, ResponsesAPIResponse) or not is_background_response(response): - return + return None if response.status not in TERMINAL_RESPONSE_STATUSES: - return + return None managed_files_hook: Final = _managed_files_hook(proxy_logging_obj) track_untracked: Final = ( partial( @@ -928,12 +998,11 @@ async def _bill_finished_background_response( if managed_files_hook is not None else None ) - charge: Final = await _settle_background_billing( + return await _settle_background_billing( (lambda cost_check: cost_check.bill_if_finished(response_id, response, track_untracked)) if read_settles_billing else (lambda cost_check: cost_check.bill_from_deployment(response_id, track_untracked)) ) - _report_charge(charge, fastapi_response) class _BilledCharge(Protocol): @@ -1083,27 +1152,53 @@ def _streamed_background_data_generator( user_api_key_dict=user_api_key_dict, request_tags=_background_response_spend_tags(request_data), ) - return _bill_streamed_background_response(response, chunks, track_untracked) + return _bill_streamed_background_response( + response, chunks, partial(_bill_finished_stream, track_untracked=track_untracked) + ) + + +def _streamed_retrieval_data_generator( + select_data_generator: Callable[..., AsyncIterator[str | bytes]], + response_id: str, + read_settles_billing: bool, + response: object, + user_api_key_dict: UserAPIKeyAuth, + request_data: Mapping[str, object], + request: Request | None = None, +) -> AsyncIterator[str | bytes]: + chunks: Final = select_data_generator( + response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, request=request + ) + return _bill_streamed_background_response( + response, + chunks, + partial( + _settle_finished_background_read, + response_id=response_id, + user_api_key_dict=user_api_key_dict, + read_settles_billing=read_settles_billing, + ), + ) async def _bill_streamed_background_response( stream: object, chunks: AsyncIterator[str | bytes], - track_untracked: Callable[[str, ResponsesAPIResponse], Awaitable[bool]], + bill: Callable[[ResponsesAPIResponse], Awaitable[object]], ) -> AsyncIterator[str | bytes]: async for chunk in chunks: completed: object = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream) if not isinstance(completed, ResponsesAPIResponse): yield chunk continue - await _bill_finished_stream(completed, track_untracked) + await bill(completed) yield chunk async for remaining_chunk in chunks: yield remaining_chunk return late_completed: Final = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream) if isinstance(late_completed, ResponsesAPIResponse): - await _bill_finished_stream(late_completed, track_untracked) + await bill(late_completed) async def _bill_finished_stream( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0b91e57f01e..08840b44532 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -7,6 +7,7 @@ import time from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain from types import MappingProxyType from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast @@ -39,7 +40,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model, tokenizer_models from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router @@ -1599,7 +1600,19 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) + counted_as: Final = MappingProxyType({model: tokenizer_models(model, llm_router) for model in models}) + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=tuple(dict.fromkeys(chain.from_iterable(counted_as.values()))), + ) + return MappingProxyType( + { + model: count + for model, names in counted_as.items() + if (count := max((counts[name] for name in names if name in counts), default=None)) is not None + } + ) def _estimate_input_tokens( diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 927152296e9..0139869c6c6 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -15,14 +15,16 @@ import asyncio import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final +from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.token_counter import served_openai_models, unresolved_tokenizer_model from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.token_counter import ( @@ -33,6 +35,9 @@ from litellm.rust_bridge.token_counter import ( rust_tokenizer, ) +if TYPE_CHECKING: + from litellm.router import Router + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -47,6 +52,25 @@ def _approximate_input_size(request_body: Mapping[str, object]) -> int: return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) +def tokenizer_models(model: str, llm_router: Router | None) -> tuple[str, ...]: + """The names to count a request for `model` with, keeping the largest count. + + An alias resolves to its target group. A group served only by OpenAI models counts as those models, so it gets + their tokenizer. Any other group counts as its OpenAI models, if any, and as the requested name with the encoding + tiktoken gives that name as written, so its other deployments keep that count without loading their vendor's + tokenizer. A name the router does not index counts as written.""" + if llm_router is None: + return (model,) + group: Final = resolve_model_group_alias(llm_router.model_group_alias, model) or model + served: Final = served_openai_models(llm_router.deployments_named(group) or ()) + if not served: + return (model,) + openai_served: Final = tuple(served_model for served_model in served if served_model is not None) + if len(openai_served) == len(served): + return openai_served + return (unresolved_tokenizer_model(model), *openai_served) + + async def count_input_tokens( request_body: dict, raw_body: bytes | None, diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 89733429927..2fa837cffba 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -227,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -294,6 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -738,6 +741,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..9f58c267231 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -168,7 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import ( record_served_output_texts, served_stream_output_texts, ) -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model from litellm.llms import load_guardrail_translation_mappings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( @@ -1139,7 +1139,9 @@ def _estimated_prompt_usage( None, ) return _estimate_dispatched_failure_usage( - model=str(model_call_details.get("model") or ""), + model=usage_tokenizer_model( + str(model_call_details.get("model") or ""), get_base_model_from_metadata(model_call_details) + ), request_input=( messages_since_newest_compaction(dispatched_input) if dispatched_call_type in _RESPONSES_CALL_TYPES diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 7b9068ff95e..e9b534bdd89 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,12 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ( + ModelInfoLookupMiss, + ProviderConfigManager, + lookup_model_info_for_registration, + resolve_api_key_secret_reference, +) from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -42,8 +47,13 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions -from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.realtime.handler import ( + OpenAIRealtime, + confirm_gateway_session_started, + confirm_session_started, +) from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config @@ -740,6 +750,18 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) +_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live") +_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0 + + +def _cost_map_lists_outside_realtime(model: str) -> bool: + model_info: Final = lookup_model_info_for_registration(model, "openai") + endpoints: Final = ( + () if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or () + ) + return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints) + + _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} @@ -785,6 +807,7 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, + gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -800,9 +823,16 @@ async def realtime_health_check( without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: - bool - True if connection is successful, False otherwise + bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an + error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports + the failure in its first server event. Behind any other OpenAI api_base the first event is read for + `gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy. + An `invalid_model` error also counts as healthy unless litellm's + cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live + and transcription models are served through other endpoints. The listed `mode` is not read, since a + deployment's own `mode: realtime` replaces it when the router registers the deployment Raises: - Exception - if the connection is not successful + Exception - if the connection is not successful or if OpenAI's first server event is any other error """ import websockets @@ -822,10 +852,11 @@ async def realtime_health_check( str | None, api_version or resolved_params.get("api_version") ) url: str | None = None + openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/" health_check_api_key: Final = ( await resolve_openai_bearer_token_async( static_api_key=openai_static_api_key(resolved_api_key), - api_base=resolved_api_base, + api_base=openai_api_base, litellm_params=resolved_params, ) if custom_llm_provider == LlmProviders.OPENAI.value @@ -850,10 +881,7 @@ async def realtime_health_check( query_params=azure_query_params, ) elif custom_llm_provider == "openai": - url = openai_realtime.construct_url( - api_base=resolved_api_base or "https://api.openai.com/", - query_params={"model": model}, - ) + url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model}) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} @@ -899,8 +927,23 @@ async def realtime_health_check( additional_headers=auth_headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=realtime_ssl_for_url(url), - ): - return True + ) as connection: + if custom_llm_provider != "openai": + return True + if not is_openai_backed_api_base(url): + return await confirm_gateway_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + first_event_wait_seconds=gateway_first_event_wait_seconds, + ) + return await confirm_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + ) _realtime_health_check = realtime_health_check diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 4b897ff58c0..ce530447c0d 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,5 +1,8 @@ -from collections.abc import Sequence +import json +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass +from itertools import groupby +from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools from pydantic import ValidationError @@ -15,8 +18,8 @@ class _InputItemType(LiteLLMBaseModel): type: str = "" -class _AdditionalToolsItem(LiteLLMBaseModel): - tools: tuple[dict[str, object], ...] = () +class _ToolList(LiteLLMBaseModel): + tools: tuple[Mapping[str, object], ...] = () @dataclass(frozen=True, slots=True) @@ -33,12 +36,54 @@ def _is_additional_tools_item(item: object) -> bool: return False -def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: +def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]: try: - parsed: Final = _AdditionalToolsItem.model_validate(item) + return _ToolList.model_validate(value).tools except ValidationError: return () - return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in parsed.tools) + + +def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in _tool_list_of(item)) + + +def _tool_identity(tool: Mapping[str, object]) -> str: + name: Final = tool.get("name") + if isinstance(name, str) and name: + return f"{tool.get('type')}:{name}" + return json.dumps(tool, sort_keys=True, default=str) + + +def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mapping[str, object]]: + for namespace in copies: + yield from _tool_list_of(namespace) + + +def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + newest: Final = copies[-1] + if newest.get("type") != "namespace" or len(copies) == 1: + return newest + members: Final = tuple(_members_of_copies(copies)) + return {**newest, "tools": list(_unique_tools(members))} if members else newest + + +def _identity_of(keyed_tool: tuple[str, Mapping[str, object]]) -> str: + return keyed_tool[0] + + +def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + """Codex re-sends its tool list in the input on every turn, so the same tool arrives many times. + Keep each tool once, at its first position, with the newest copy's definition, and merge the + members of a namespace sent more than once.""" + keyed: Final = tuple((_tool_identity(tool), tool) for tool in tools) + copies_by_identity: Final = MappingProxyType( + { + identity: tuple(tool for _, tool in group) + for identity, group in groupby(sorted(keyed, key=_identity_of), key=_identity_of) + } + ) + first_seen_order: Final = tuple(dict.fromkeys(identity for identity, _ in keyed)) + return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order) def hoist_additional_tools( @@ -58,4 +103,8 @@ def hoist_additional_tools( len(items), ) remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)] - return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted) + unique: Final = tuple( + cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts + for tool in _unique_tools((*existing, *hoisted)) + ) + return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted) diff --git a/litellm/responses/litellm_completion_transformation/client_tool_search.py b/litellm/responses/litellm_completion_transformation/client_tool_search.py new file mode 100644 index 00000000000..030bb014b7a --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/client_tool_search.py @@ -0,0 +1,268 @@ +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # loaded tool specs are forwarded as the client sent them + +from openai.types.responses.response_input_param import ( + FunctionCallOutput, + ResponseFunctionToolCallParam, + ResponseInputItemParam, +) +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openai import ( + ALL_RESPONSES_API_TOOL_PARAMS, + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, + ResponseInputParam, +) + +from .custom_tools import function_chat_tool +from .tool_search import CLIENT_TOOL_SEARCH_FUNCTION_NAME, has_client_tool_search + +__all__ = [ + "ClientToolSearchBridge", + "bridge_client_tool_search", + "client_tool_search_chat_tool", + "client_tool_search_loaded_tools", +] + +_DEFAULT_SEARCH_DESCRIPTION: Final = ( + "Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn." +) +_DEFAULT_SEARCH_PARAMETERS: Final = MappingProxyType( + { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for deferred tools."}, + "limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."}, + }, + "required": ["query"], + "additionalProperties": False, + } +) + + +class _ClientToolSearchItem(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str = "" + execution: str | None = None + call_id: str | None = None + arguments: object = None + tools: tuple[Mapping[str, object], ...] = () + + +class _ToolSpec(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str = "" + name: str = "" + description: str = "" + defer_loading: object = None + tools: tuple[Mapping[str, object], ...] = () + + +class _ClientToolSearchTool(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + description: str = "" + parameters: Mapping[str, object] | None = None + + +@dataclass(frozen=True, slots=True) +class _LoadedTool: + namespace: str | None + namespace_description: str + spec: Mapping[str, object] + + @property + def callable_name(self) -> str: + name: Final = str(self.spec.get("name") or "") + return name if self.namespace is None else f"{self.namespace}__{name}" + + +@dataclass(frozen=True, slots=True) +class ClientToolSearchBridge: + input: str | ResponseInputParam + tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...] + bridged: bool + + +def client_tool_search_chat_tool(tool: Mapping[str, object]) -> ChatCompletionToolParam: + spec: Final = _ClientToolSearchTool.model_validate(tool) + function: Final = ChatCompletionToolParamFunctionChunk( + name=CLIENT_TOOL_SEARCH_FUNCTION_NAME, + description=spec.description or _DEFAULT_SEARCH_DESCRIPTION, + parameters=dict(spec.parameters or _DEFAULT_SEARCH_PARAMETERS), + ) + return function_chat_tool(function, None, False) + + +def _client_tool_search_item(item: object) -> _ClientToolSearchItem | None: + try: + parsed: Final = _ClientToolSearchItem.model_validate(item) + except ValidationError: + return None + if parsed.type not in ("tool_search_call", "tool_search_output") or parsed.execution != "client": + return None + return parsed + + +_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) + + +def _searched_tools(item: object) -> tuple[Mapping[str, object], ...]: + parsed: Final = _client_tool_search_item(item) + if parsed is None or parsed.type != "tool_search_output" or not parsed.call_id: + return () + return parsed.tools + + +def client_tool_search_loaded_tools(input: object) -> tuple[Mapping[str, object], ...]: + if not isinstance(input, (list, tuple)): + return () + return tuple(chain.from_iterable(map(_searched_tools, _INPUT_ITEMS.validate_python(input)))) + + +def _tool_spec(tool: object) -> _ToolSpec | None: + try: + return _ToolSpec.model_validate(tool) + except ValidationError: + return None + + +def _as_tool_param(spec: Mapping[str, object]) -> ALL_RESPONSES_API_TOOL_PARAMS: + return cast("ALL_RESPONSES_API_TOOL_PARAMS", spec) # cast-ok: client tool specs are forwarded as sent + + +def _without_deferral(spec: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in spec.items() if key != "defer_loading"} + + +def _loaded_tools(tool: Mapping[str, object]) -> tuple[_LoadedTool, ...]: + spec: Final = _tool_spec(tool) + if spec is None: + return () + if spec.type != "namespace": + return (_LoadedTool(namespace=None, namespace_description="", spec=_without_deferral(tool)),) + return tuple( + _LoadedTool(namespace=spec.name, namespace_description=spec.description, spec=_without_deferral(member)) + for member in spec.tools + ) + + +def _all_loaded_tools(tools: Iterable[Mapping[str, object]]) -> tuple[_LoadedTool, ...]: + return tuple(chain.from_iterable(map(_loaded_tools, tools))) + + +def _callable_names(tool: object) -> tuple[str, ...]: + spec: Final = _tool_spec(tool) + if spec is None: + return () + if spec.type != "namespace": + return (spec.name,) + return tuple(f"{spec.name}__{member.get('name') or ''}" for member in spec.tools) + + +def _first_occurrences(loaded: Sequence[_LoadedTool], taken: frozenset[str]) -> tuple[_LoadedTool, ...]: + first_index: Final = {tool.callable_name: index for index, tool in reversed(tuple(enumerate(loaded)))} + return tuple( + tool + for index, tool in enumerate(loaded) + if first_index[tool.callable_name] == index and tool.callable_name not in taken + ) + + +def _regrouped(loaded: Sequence[_LoadedTool]) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + namespaces: Final = {tool.namespace: tool.namespace_description for tool in reversed(loaded) if tool.namespace} + ordered_namespaces: Final = tuple(dict.fromkeys(tool.namespace for tool in loaded if tool.namespace)) + top_level: Final = tuple(tool.spec for tool in loaded if tool.namespace is None) + grouped: Final = tuple( + { + "type": "namespace", + "name": namespace, + "description": namespaces[namespace], + "tools": tuple(tool.spec for tool in loaded if tool.namespace == namespace), + } + for namespace in ordered_namespaces + ) + return tuple(_as_tool_param(tool) for tool in (*top_level, *grouped)) + + +def _is_deferred(tool: object) -> bool: + spec: Final = _tool_spec(tool) + return spec is not None and spec.defer_loading is True + + +def _without_deferred(tool: ALL_RESPONSES_API_TOOL_PARAMS) -> ALL_RESPONSES_API_TOOL_PARAMS | None: + spec: Final = _tool_spec(tool) + if spec is None: + return tool + if spec.type != "namespace": + return None if spec.defer_loading is True else tool + eager_members: Final = tuple(member for member in spec.tools if not _is_deferred(member)) + if len(eager_members) == len(spec.tools): + return tool + if not eager_members: + return None + return _as_tool_param({**tool, "tools": eager_members}) + + +def _search_output_text(loaded: Sequence[_LoadedTool]) -> str: + if not loaded: + return "No tools matched the search." + return "Loaded these tools, which are now callable: " + ", ".join(tool.callable_name for tool in loaded) + + +def _search_arguments_json(arguments: object) -> str: + return arguments if isinstance(arguments, str) else json.dumps(arguments or {}) + + +def _bridged_item(item: ResponseInputItemParam) -> ResponseInputItemParam | None: + parsed: Final = _client_tool_search_item(item) + if parsed is None: + return item + if not parsed.call_id: + return None + if parsed.type == "tool_search_call": + return ResponseFunctionToolCallParam( + type="function_call", + call_id=parsed.call_id, + name=CLIENT_TOOL_SEARCH_FUNCTION_NAME, + arguments=_search_arguments_json(parsed.arguments), + ) + return FunctionCallOutput( + type="function_call_output", call_id=parsed.call_id, output=_search_output_text(_all_loaded_tools(parsed.tools)) + ) + + +def bridge_client_tool_search( + input: str | ResponseInputParam, + tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, + loaded_earlier: Sequence[Mapping[str, object]] = (), +) -> ClientToolSearchBridge: + existing: Final = tuple(tools or ()) + searches_on_client: Final = has_client_tool_search(existing) + items: Final = () if isinstance(input, str) else tuple(input) + search_items: Final = tuple(parsed for item in items if (parsed := _client_tool_search_item(item)) is not None) + if not searches_on_client and not search_items: + return ClientToolSearchBridge(input=input, tools=existing, bridged=False) + eager: Final = ( + tuple(kept for tool in existing if (kept := _without_deferred(tool)) is not None) + if searches_on_client + else existing + ) + loaded: Final = _all_loaded_tools((*loaded_earlier, *client_tool_search_loaded_tools(input))) + taken: Final = frozenset(chain.from_iterable(map(_callable_names, eager))) + bridged_input: Final = ( + input if isinstance(input, str) else [bridged for item in input if (bridged := _bridged_item(item)) is not None] + ) + return ClientToolSearchBridge( + input=bridged_input, + tools=(*eager, *_regrouped(_first_occurrences(loaded, taken))), + bridged=True, + ) diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 7679b4ede71..8e8e4066bb8 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.responses.main import previous_response_client_ids -from litellm.types.llms.openai import ResponseInputParam +from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam from litellm.utils import ProviderConfigManager _COUNTED_CHAT_PARAMS: Final = ( @@ -119,14 +119,13 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M if not isinstance(responses_input, list): return responses_request input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does - hoisted: Final = hoist_additional_tools(input_items, None) + top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does + "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ()) + ) + hoisted: Final = hoist_additional_tools(input_items, top_level_tools) if not hoisted.hoisted: return responses_request - return { - **responses_request, - "input": hoisted.input, - "tools": [*_ITEMS.validate_python(responses_request.get("tools") or ()), *hoisted.hoisted], - } + return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index ea3bda1b8b5..141e97d902d 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -2,17 +2,21 @@ Handler for transforming responses api requests to litellm.completion requests """ -from collections.abc import Coroutine, Mapping +from collections.abc import Coroutine, Mapping, Sequence +from dataclasses import dataclass from typing import Final from pydantic import TypeAdapter import litellm -from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools +from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary +from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, ) +from litellm.responses.litellm_completion_transformation.tool_search import has_client_tool_search from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, with_compaction_items_as_params, @@ -28,6 +32,28 @@ from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +@dataclass(frozen=True, slots=True) +class _BridgedRequest: + input: str | ResponseInputParam + responses_api_request: ResponsesAPIOptionalRequestParams + + +def _bridged_request( + hoisted: HoistedAdditionalTools, + responses_api_request: ResponsesAPIOptionalRequestParams, + loaded_earlier: Sequence[Mapping[str, object]], +) -> _BridgedRequest: + searched: Final = bridge_client_tool_search(hoisted.input, hoisted.tools, loaded_earlier) + return _BridgedRequest( + input=searched.input, + responses_api_request=( + {**responses_api_request, "tools": list(searched.tools)} + if hoisted.hoisted or searched.bridged + else responses_api_request + ), + ) + + class LiteLLMCompletionTransformationHandler: def response_api_handler( self, @@ -48,10 +74,22 @@ class LiteLLMCompletionTransformationHandler: | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] ): hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools")) - bridged_input: Final = hoisted.input - bridged_request: Final[ResponsesAPIOptionalRequestParams] = ( - {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request - ) + previous_response_id: Final = responses_api_request.get("previous_response_id") + if _is_async and previous_response_id and has_client_tool_search(hoisted.tools): + return self._async_response_api_handler_after_earlier_searches( + previous_response_client_ids, + previous_response_id, + hoisted, + model=model, + responses_api_request=responses_api_request, + custom_llm_provider=custom_llm_provider, + stream=stream, + extra_headers=extra_headers, + request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + ) + bridged: Final = _bridged_request(hoisted, responses_api_request, ()) + bridged_input: Final = bridged.input + bridged_request: Final = bridged.responses_api_request litellm_completion_request: Final[dict] = ( LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, @@ -73,13 +111,8 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) - completion_args: Final = {} - completion_args.update(kwargs) - completion_args.update(litellm_completion_request) - completion_args["_skip_responses_api_bridge"] = True - litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **completion_args, + **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): @@ -104,9 +137,53 @@ class LiteLLMCompletionTransformationHandler: ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") + async def _async_response_api_handler_after_earlier_searches( + self, + previous_response_client_ids: tuple[str, ...], + previous_response_id: str, + hoisted: HoistedAdditionalTools, + /, + *, + model: str, + responses_api_request: ResponsesAPIOptionalRequestParams, + custom_llm_provider: str | None, + stream: bool | None, + extra_headers: Mapping[str, object] | None, + request_kwargs: Mapping[str, object], + ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + stored_turns: Final = await ResponsesSessionHandler.get_stored_turns_for_previous_response_id( + previous_response_id, previous_response_client_ids + ) + bridged: Final = _bridged_request( + hoisted, + responses_api_request, + ResponsesSessionHandler.client_tool_search_tools_loaded_before( + stored_turns, previous_response_id, previous_response_client_ids + ), + ) + return await self.async_response_api_handler( + previous_response_client_ids, + stored_turns, + litellm_completion_request=( + LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=bridged.input, + responses_api_request=bridged.responses_api_request, + custom_llm_provider=custom_llm_provider, + stream=stream, + extra_headers=extra_headers, + **request_kwargs, + ) + ), + request_input=bridged.input, + responses_api_request=bridged.responses_api_request, + **request_kwargs, + ) + async def async_response_api_handler( self, previous_response_client_ids: tuple[str, ...] = (), + stored_turns: StoredTurns | None = None, /, *, litellm_completion_request: dict, @@ -122,16 +199,12 @@ class LiteLLMCompletionTransformationHandler: instructions=responses_api_request.get("instructions"), request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), previous_response_client_ids=previous_response_client_ids, + stored_turns=stored_turns, input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) - acompletion_args: Final = {} - acompletion_args.update(kwargs) - acompletion_args.update(litellm_completion_request) - acompletion_args["_skip_responses_api_bridge"] = True - litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **acompletion_args, + **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 1ab6293b432..ce2396355a2 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -9,6 +9,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler +from litellm.responses.litellm_completion_transformation.client_tool_search import ( + bridge_client_tool_search, + client_tool_search_loaded_tools, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( holds_compaction_summary, input_items, @@ -88,6 +92,12 @@ class _ReplayStart: at_output: bool +@dataclass(frozen=True, slots=True) +class StoredTurns: + spend_logs: tuple["SpendLogsPayload", ...] + proxy_server_requests: tuple[Mapping[str, object] | None, ...] + + def _compaction_point( index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None ) -> _ReplayStart | None: @@ -190,6 +200,7 @@ class ResponsesSessionHandler: previous_response_id: str, item_cache_control: bool = False, client_response_ids: tuple[str, ...] = (), + stored_turns: StoredTurns | None = None, ) -> ChatCompletionSession: """ Return the chat completion message history for a previous response id @@ -200,23 +211,21 @@ class ResponsesSessionHandler: ) verbose_proxy_logger.debug("inside get_chat_completion_message_history_for_previous_response_id") - all_spend_logs: Final = [ - spend_log - for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + turns: Final = ( + stored_turns + if stored_turns is not None + else await ResponsesSessionHandler.get_stored_turns_for_previous_response_id( previous_response_id, client_response_ids ) - if _is_a_conversation_turn(spend_log) - ] + ) + all_spend_logs: Final = turns.spend_logs verbose_proxy_logger.debug("found %s spend logs for this response id", len(all_spend_logs)) litellm_session_id: str | None = None if len(all_spend_logs) > 0: litellm_session_id = all_spend_logs[0].get("session_id") - proxy_server_requests: Final = [ - await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log) - for spend_log in all_spend_logs - ] + proxy_server_requests: Final = turns.proxy_server_requests replayable_turns: Final = _turns_through_referenced_response( all_spend_logs, _referenced_request_ids(previous_response_id, client_response_ids) ) @@ -252,6 +261,45 @@ class ResponsesSessionHandler: instructions=_latest_replayable_instructions(proxy_server_requests), ) + @staticmethod + async def get_stored_turns_for_previous_response_id( + previous_response_id: str, + client_response_ids: tuple[str, ...] = (), + ) -> StoredTurns: + spend_logs: Final = tuple( + spend_log + for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + previous_response_id, client_response_ids + ) + if _is_a_conversation_turn(spend_log) + ) + return StoredTurns( + spend_logs=spend_logs, + proxy_server_requests=tuple( + [ + await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log) + for spend_log in spend_logs + ] + ), + ) + + @staticmethod + def client_tool_search_tools_loaded_before( + stored_turns: StoredTurns, + previous_response_id: str, + client_response_ids: tuple[str, ...] = (), + ) -> tuple[Mapping[str, object], ...]: + turns: Final = _turns_through_referenced_response( + stored_turns.spend_logs, _referenced_request_ids(previous_response_id, client_response_ids) + ) + return tuple( + chain.from_iterable( + client_tool_search_loaded_tools(request.get("input")) + for request in stored_turns.proxy_server_requests[:turns] + if request is not None + ) + ) + @staticmethod def _history_of_turns( spend_logs: Sequence["SpendLogsPayload"], @@ -310,7 +358,7 @@ class ResponsesSessionHandler: if response_input_param: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=response_input_param, + input=bridge_client_tool_search(response_input_param, None).input, responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, @@ -325,7 +373,7 @@ class ResponsesSessionHandler: if (responses_api_output := _stored_responses_api_output(_response_output)) is not None: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=responses_api_output, + input=bridge_client_tool_search(responses_api_output, None).input, responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 982158040c4..892da541935 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,7 +1,8 @@ import time import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from itertools import filterfalse +from types import MappingProxyType from typing import Any, Final, cast import litellm @@ -18,9 +19,9 @@ from litellm.responses.litellm_completion_transformation.custom_tools import ( ) from litellm.responses.litellm_completion_transformation.reasoning_items import mint_reasoning_item_id from litellm.responses.litellm_completion_transformation.tool_search import ( + ToolSearchExecution, build_tool_search_call, - has_server_tool_search, - is_anthropic_tool_search_use, + tool_search_routing, ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, @@ -30,6 +31,7 @@ from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PART_UNION_TYPES, BaseLiteLLMOpenAIResponseObject, + ChatCompletionAnnotation, ContentPartAddedEvent, ContentPartDoneEvent, ContentPartDonePartOutputText, @@ -56,6 +58,7 @@ from litellm.types.llms.openai import ( WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) +from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -105,7 +108,6 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool: getattr(delta, "reasoning_content", None), delta.tool_calls, delta.function_call, - getattr(delta, "annotations", None), getattr(delta, "images", None), getattr(delta, "audio", None), ) @@ -150,7 +152,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.sent_output_text_done_event: bool = False self.sent_output_content_part_done_event: bool = False self.sent_output_item_done_event: bool = False - self.sent_annotation_events: bool = False + self._annotation_index: int = 0 + self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = () self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None self.completed_response = None self.final_text: str = "" @@ -190,16 +193,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools) self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids - self._tool_search_requested: bool = has_server_tool_search(request_tools) - self._tool_search_call_ids: frozenset[str] = frozenset() + self._tool_search = tool_search_routing(request_tools) + self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({}) - def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool: - if call_id in self._tool_search_call_ids: - return True - if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name): - return False - self._tool_search_call_ids = self._tool_search_call_ids | {call_id} - return True + async def aclose(self) -> None: + await self.litellm_custom_stream_wrapper.aclose() + + def _tool_search_execution(self, call_id: str, fn_name: str) -> ToolSearchExecution | None: + known: Final = self._tool_search_executions.get(call_id) + if known is not None: + return known + execution: Final = self._tool_search.execution_of(call_id, fn_name) + if execution is None: + return None + self._tool_search_executions = MappingProxyType({**self._tool_search_executions, call_id: execution}) + return execution def _next_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> OutputItemAddedEvent: self._sequence_number += 1 @@ -214,18 +222,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None: self._pending_tool_events.append(self._next_output_item_added(output_index, item)) - def _queue_tool_search_call_added(self, call_id: str, output_index: int, args_delta: str) -> None: + def _queue_tool_search_call_added( + self, call_id: str, output_index: int, args_delta: str, execution: ToolSearchExecution + ) -> None: if call_id not in self._tool_args_by_call_id: self._tool_args_by_call_id[call_id] = "" - item: Final = build_tool_search_call(call_id, "", "in_progress") + item: Final = build_tool_search_call(call_id, "", "in_progress", execution) self._tool_item_id_by_call_id[call_id] = item.id self._queue_output_item_added(output_index, BaseLiteLLMOpenAIResponseObject(**item.model_dump())) self._tool_args_by_call_id[call_id] += args_delta - def _queue_tool_search_call_done(self, call_id: str, output_index: int, final_args: str) -> None: - self._queue_tool_search_call_added(call_id, output_index, "") + def _queue_tool_search_call_done( + self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution + ) -> None: + self._queue_tool_search_call_added(call_id, output_index, "", execution) item: Final = build_tool_search_call( - call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed" + call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution ) self._sequence_number += 1 self._pending_tool_events.append( @@ -339,7 +351,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): namespace_kwargs: Final = {"namespace": tool_namespace} if tool_namespace else {} return {**item_kwargs, "name": tool_name, **namespace_kwargs} - def _is_reasoning_end(self, chunk): + def _is_reasoning_end(self, chunk: ModelResponseStream) -> bool: if not chunk.choices: return False delta: Final = chunk.choices[0].delta @@ -350,7 +362,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if hasattr(delta, "thinking_blocks") and delta.thinking_blocks: return False - return delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None + return bool( + delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None + ) def _reserve_web_search_indexes(self, provider_fields: object) -> None: if not isinstance(provider_fields, dict): @@ -428,10 +442,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): else: fn_name = str(getattr(fn, "name", "") or "") fn_args_delta = serialize_tool_call_arguments(getattr(fn, "arguments", "")) + if call_id not in self._tool_output_index_by_call_id: + self._queue_held_annotations() output_index = self._get_or_assign_tool_output_index(call_id) - if self._is_tool_search_call(call_id, fn_name): - self._queue_tool_search_call_added(call_id, output_index, fn_args_delta) + if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None: + self._queue_tool_search_call_added(call_id, output_index, fn_args_delta, search_execution) continue self._started_tool_call = True @@ -499,8 +515,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._queue_web_search_events(call_id, web_search_call) self._queued_web_search_call_ids.add(call_id) continue - if self._is_tool_search_call(call_id, fn_name): - self._queue_tool_search_call_done(call_id, output_index, fn_args) + if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None: + self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution) continue # Track if this is a new tool call that wasn't streamed @@ -782,8 +798,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return event def _queue_message_item_added_events(self) -> None: - if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()}" + item_id: Final = self._message_item_id() self.sent_message_item_added_event = True self.sent_content_part_added_event = True self._sequence_number += 1 @@ -792,7 +807,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): output_index=self._message_index(), item=BaseLiteLLMOpenAIResponseObject( **{ - "id": self._cached_item_id, + "id": item_id, "type": "message", "role": "assistant", "status": "in_progress", @@ -1081,6 +1096,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self.litellm_model_response: self._queue_compaction_item_added() self._queue_compaction_item_done() + self._queue_held_annotations() if self._pending_response_events: return self._pending_response_events.pop(0) # If tool calls exist, emit tool events before finishing/response.completed. @@ -1163,7 +1179,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): delta: Final = chunk.choices[0].delta if chunk.choices else None if delta and getattr(delta, "reasoning_content", None): self._accumulated_reasoning_content_parts.append(delta.reasoning_content) - if not self._is_reasoning_end(chunk): + if self._is_reasoning_end(chunk): + self._queue_reasoning_done_events() + + def _queue_reasoning_done_events(self) -> None: + if not self._reasoning_active or self._reasoning_done_emitted: return reasoning_content: Final = "".join(self._accumulated_reasoning_content_parts) self._cached_reasoning_item_id = ( @@ -1237,9 +1257,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._snapshot_chunk_for_stream_chunk_builder(chunk) ) self._queue_reasoning_done_events_on_reasoning_end(chunk) - response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) - if response_api_chunk: - self._pending_response_events.append(response_api_chunk) + self._queue_chunk_events(chunk) if self._pending_response_events: return self._pending_response_events.pop(0) @@ -1300,9 +1318,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._snapshot_chunk_for_stream_chunk_builder(cast(ModelResponseStream, chunk)) ) self._queue_reasoning_done_events_on_reasoning_end(chunk) - response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) - if response_api_chunk: - self._pending_response_events.append(response_api_chunk) + self._queue_chunk_events(chunk) if self._pending_response_events: return self._pending_response_events.pop(0) # Otherwise, loop to next chunk @@ -1313,6 +1329,58 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.finished = True raise e + def _queue_chunk_events(self, chunk: ModelResponseStream) -> None: + self._hold_chunk_annotations(chunk) + response_api_chunk: Final = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) + if response_api_chunk: + self._pending_response_events.append(response_api_chunk) + if self.sent_message_item_added_event: + self._queue_held_annotations() + + def _hold_chunk_annotations(self, chunk: ModelResponseStream) -> None: + chunk_annotations: Final = (getattr(chunk.choices[0].delta, "annotations", None) or ()) if chunk.choices else () + self._annotations_awaiting_message = (*self._annotations_awaiting_message, *chunk_annotations) + + def _queue_held_annotations(self) -> None: + annotations: Final = self._annotations_awaiting_message + if not annotations: + return + self._annotations_awaiting_message = () + self._queue_reasoning_done_events() + self._queue_annotation_added_events(annotations) + + def _queue_annotation_added_events(self, annotations: Sequence[ChatCompletionAnnotation]) -> None: + if not self.sent_message_item_added_event: + self._queue_message_item_added_events() + item_id: Final = self._message_item_id() + response_annotations: Final = ( + LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( + annotations=annotations + ) + ) + self._pending_response_events.extend( + self._annotation_added_event(item_id, annotation) for annotation in response_annotations + ) + + def _message_item_id(self) -> str: + if self._cached_item_id is None: + self._cached_item_id = f"msg_{uuid.uuid4()}" + return self._cached_item_id + + def _annotation_added_event( + self, item_id: str, annotation: GenericResponseOutputItemContentAnnotation + ) -> OutputTextAnnotationAddedEvent: + event: Final = OutputTextAnnotationAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, + item_id=item_id, + output_index=self._message_index(), + content_index=0, + annotation_index=self._annotation_index, + annotation=annotation.model_dump(), + ) + self._annotation_index += 1 + return event + def _transform_chat_completion_chunk_to_response_api_chunk( self, chunk: ModelResponseStream ) -> ResponsesAPIStreamingResponse | None: @@ -1321,37 +1389,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): This currently handles emitting the OutputTextDeltaEvent, which is used by other tools using the responses API and the ReasoningSummaryTextDeltaEvent, which is used by the responses API to emit reasoning content. - It also handles emitting annotation.added events when annotations are detected in the chunk. """ - if self._cached_item_id is None: - self._cached_item_id = f"msg_{uuid.uuid4()}" - item_id: Final = self._cached_item_id + item_id: Final = self._message_item_id() - # Check if this chunk has annotations first (before processing text/reasoning) - # This ensures we detect and queue annotation events from the annotation chunk - if chunk.choices and hasattr(chunk.choices[0].delta, "annotations"): - annotations: Final = chunk.choices[0].delta.annotations - if annotations and self.sent_annotation_events is False: - self.sent_annotation_events = True - # Store annotation events to emit them one by one - if not hasattr(self, "_pending_annotation_events"): - response_annotations = LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( - annotations=annotations - ) - self._pending_annotation_events = [] - for idx, annotation in enumerate(response_annotations): - annotation_dict = ( - annotation.model_dump() if hasattr(annotation, "model_dump") else dict(annotation) - ) - event = OutputTextAnnotationAddedEvent( - type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, - item_id=item_id, - output_index=self._message_index(), - content_index=0, - annotation_index=idx, - annotation=annotation_dict, - ) - self._pending_annotation_events.append(event) # Priority 1: Handle reasoning content (highest priority) if ( chunk.choices @@ -1395,13 +1435,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._pending_tool_events: return self._pending_tool_events.pop(0) - # Priority 4: If we have pending annotation events, emit the next one - # This happens when the current chunk has no text/reasoning content - if hasattr(self, "_pending_annotation_events") and self._pending_annotation_events: - event = self._pending_annotation_events.pop(0) - return event - - # Priority 5: If we have pending tool events (from earlier chunk), emit the next one if self._pending_tool_events: return self._pending_tool_events.pop(0) @@ -1444,7 +1477,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: return False - return not any(getattr(part, "text", None) for part in attribute_items(item, "content")) + return not any( + getattr(part, "text", None) or getattr(part, "annotations", None) + for part in attribute_items(item, "content") + ) def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: if not isinstance(self.litellm_model_response, ModelResponse): diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 780a23e02b2..5a1b27f9092 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -1,4 +1,5 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set +from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -18,25 +19,31 @@ if TYPE_CHECKING: __all__ = [ "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", + "CLIENT_TOOL_SEARCH_FUNCTION_NAME", "CompletionRoute", "ToolSearchCallStatus", + "ToolSearchExecution", + "ToolSearchRouting", "answers_tool_search", "anthropic_tool_search_chat_tool", "build_tool_search_call", "chat_config_builds_anthropic_request", "chat_messages_without_tool_search_calls", "chat_route_rejects_unanswered_tool_calls", + "has_client_tool_search", "has_server_tool_search", "holds_tool_search_use", "input_without_unanswered_anthropic_tool_search_calls", "is_anthropic_tool_search_chat_tool", "is_anthropic_tool_search_use", + "is_client_tool_search_tool", "is_openai_tool_search_tool", "is_server_tool_search_tool", "provider_builds_anthropic_request", "provider_chat_config", "runs_tool_search_on_the_server", "tool_search_call_ids", + "tool_search_routing", "tools_with_anthropic_tool_search_mapped", "tools_without_anthropic_tool_search", "unanswered_tool_search_call_ids", @@ -46,7 +53,9 @@ __all__ = [ ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex" +CLIENT_TOOL_SEARCH_FUNCTION_NAME: Final = "tool_search" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] +ToolSearchExecution: TypeAlias = Literal["server", "client"] _ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -79,6 +88,35 @@ def has_server_tool_search(tools: Sequence[object] | None) -> bool: return any(is_server_tool_search_tool(tool) for tool in tools or ()) +def is_client_tool_search_tool(tool: object) -> bool: + try: + parsed: Final = _ToolSearchFields.model_validate(tool) + except ValidationError: + return False + return parsed.type == "tool_search" and parsed.execution == "client" + + +def has_client_tool_search(tools: Sequence[object] | None) -> bool: + return any(is_client_tool_search_tool(tool) for tool in tools or ()) + + +@dataclass(frozen=True, slots=True) +class ToolSearchRouting: + server: bool + client: bool + + def execution_of(self, tool_id: str, tool_name: str) -> ToolSearchExecution | None: + if self.server and is_anthropic_tool_search_use(tool_id, tool_name): + return "server" + if self.client and tool_name == CLIENT_TOOL_SEARCH_FUNCTION_NAME: + return "client" + return None + + +def tool_search_routing(tools: Sequence[object] | None) -> ToolSearchRouting: + return ToolSearchRouting(server=has_server_tool_search(tools), client=has_client_tool_search(tools)) + + def anthropic_tool_search_chat_tool() -> AnthropicToolSearchToolRegex: return AnthropicToolSearchToolRegex(type=ANTHROPIC_TOOL_SEARCH_TOOL_TYPE, name=ANTHROPIC_TOOL_SEARCH_TOOL_NAME) @@ -346,13 +384,18 @@ def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: return None -def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> OutputToolSearchCall: +def build_tool_search_call( + tool_id: str, + arguments_json: str, + status: ToolSearchCallStatus, + execution: ToolSearchExecution = "server", +) -> OutputToolSearchCall: arguments: Final = _parsed_arguments(arguments_json) return OutputToolSearchCall( id=openai_shaped_tool_call_item_id("tool_search_call", tool_id), call_id=tool_id, arguments={} if arguments is None else arguments, - execution="server", + execution=execution, status="incomplete" if arguments is None else status, type="tool_search_call", ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a342d704a9d..ce8841c9190 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -64,6 +64,7 @@ from litellm.llms.snowflake.chat.transformation import is_snowflake_claude_model from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, + StoredTurns, ) from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex from litellm.types.llms.base import CachedTokensDetails @@ -124,6 +125,7 @@ from litellm.types.utils import ( Usage, ) +from .client_tool_search import client_tool_search_chat_tool from .compaction_items import ( carries_only_compaction, compaction_output_item, @@ -151,16 +153,17 @@ from .custom_tools import ( from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( CompletionRoute, + ToolSearchRouting, answers_tool_search, anthropic_tool_search_chat_tool, build_tool_search_call, has_server_tool_search, holds_tool_search_use, is_anthropic_tool_search_chat_tool, - is_anthropic_tool_search_use, is_server_tool_search_tool, provider_builds_anthropic_request, tool_search_call_ids, + tool_search_routing, without_tool_search_calls, without_tool_search_uses, ) @@ -1342,6 +1345,7 @@ class LiteLLMCompletionResponsesConfig: request_kwargs: Mapping[str, object] = MappingProxyType({}), previous_response_client_ids: tuple[str, ...] = (), input_replaces_history: bool = False, + stored_turns: StoredTurns | None = None, ) -> dict[str, object]: """ Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages @@ -1357,6 +1361,7 @@ class LiteLLMCompletionResponsesConfig: ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id, item_cache_control=keeps_item_marks, client_response_ids=previous_response_client_ids, + stored_turns=stored_turns, ) chat_completion_session: Final = ( await lookup(previous_response_id) @@ -2818,9 +2823,11 @@ class LiteLLMCompletionResponsesConfig: if tool_type == "custom": converted: Final = convert_custom_tool_to_function_tool(tool) return ResponsesToolChatForm(chat_tools=() if converted is None else (converted,), web_search_options=None) - if tool_type == "tool_search" and validated_tool_search_execution(tool.get("execution")) != "client": + if tool_type == "tool_search" and validated_tool_search_execution(tool.get("execution")) == "client": + return ResponsesToolChatForm(chat_tools=(client_tool_search_chat_tool(tool),), web_search_options=None) + if tool_type == "tool_search": return ResponsesToolChatForm(chat_tools=(anthropic_tool_search_chat_tool(),), web_search_options=None) - if tool_type in ("computer_use", "image_generation", "local_shell", "shell", "tool_search"): + if tool_type in ("computer_use", "image_generation", "local_shell", "shell"): verbose_logger.warning( "Dropping Responses API tool of type '%s': it has no Chat Completions " "equivalent and the target provider would reject the request.", @@ -2960,7 +2967,7 @@ class LiteLLMCompletionResponsesConfig: request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None custom_tool_names: Final = extract_custom_tool_names(request_tools) namespace_tool_names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools) - tool_search_requested: Final = has_server_tool_search(request_tools) + tool_search: Final = tool_search_routing(request_tools) web_search_calls: Final = LiteLLMCompletionResponsesConfig._web_search_calls_by_call_id( chat_completion_response @@ -2977,7 +2984,7 @@ class LiteLLMCompletionResponsesConfig: if ( server_tool_call := LiteLLMCompletionResponsesConfig._server_tool_call_item( - tool_id, tool_name, tool_arguments, web_search_calls, tool_search_requested + tool_id, tool_name, tool_arguments, web_search_calls, tool_search ) ) is not None: responses_tools.append(server_tool_call) @@ -3042,14 +3049,15 @@ class LiteLLMCompletionResponsesConfig: tool_name: str, tool_arguments: str, web_search_calls: Mapping[str, ResponseFunctionWebSearch], - tool_search_requested: bool, + tool_search: ToolSearchRouting, ) -> ResponseFunctionWebSearch | OutputToolSearchCall | None: web_search_call: Final = web_search_calls.get(tool_id) if web_search_call is not None: return web_search_call - if tool_search_requested and is_anthropic_tool_search_use(tool_id, tool_name): - return build_tool_search_call(tool_id, tool_arguments, "completed") - return None + execution: Final = tool_search.execution_of(tool_id, tool_name) + if execution is None: + return None + return build_tool_search_call(tool_id, tool_arguments, "completed", execution) @staticmethod def _web_search_calls_by_call_id( @@ -3555,7 +3563,7 @@ class LiteLLMCompletionResponsesConfig: choice=choice, ) message_output_items.extend(image_generation_items) - elif choice.message.content is not None: + elif choice.message.content is not None or getattr(choice.message, "annotations", None): message_output_items.append( GenericResponseOutputItem( type="message", @@ -3629,13 +3637,13 @@ class LiteLLMCompletionResponsesConfig: return OutputText( type="output_text", - text=message.content, + text=message.content or "", annotations=transformed_annotations, ) @staticmethod def transform_chat_completion_annotations_to_response_output_annotations( - annotations: list[ChatCompletionAnnotation] | None, + annotations: Sequence[ChatCompletionAnnotation] | None, ) -> list[GenericResponseOutputItemContentAnnotation]: response_output_annotations: Final[list[GenericResponseOutputItemContentAnnotation]] = [] diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d474ed3991f..98356e9a5d4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -90,9 +90,9 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, + AgenticLoopParams, CallTypes, CostPerToken, - CustomPricingLiteLLMParams, all_litellm_params, ) from litellm.utils import ( @@ -106,7 +106,17 @@ if TYPE_CHECKING: else: MCPTool = Any -from .streaming_iterator import BaseResponsesAPIStreamingIterator +from .retrieve_stream_query import ( + RETRIEVE_INCLUDE_QUERY_KEYS, + InvalidRetrieveStreamQuery, + RetrieveStreamQuery, + parse_retrieve_stream_query, +) +from .streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) __all__ = ( "acancel_responses", @@ -192,6 +202,7 @@ def mock_responses_api_response( _STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_STR_DICT: Final = TypeAdapter(dict[str, object]) _FORWARDED_VALUE: Final = TypeAdapter(object) @@ -554,12 +565,6 @@ async def aresponses_api_with_mcp( return response -def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: - return MappingProxyType( - litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) - ) - - class _DeploymentModelInfo(BaseModel): model_config = ConfigDict(frozen=True) id: object = None @@ -1569,6 +1574,7 @@ def responses( local_vars=local_vars, use_chat_completions_api=use_chat_completions_api, ) + litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) from litellm.litellm_core_utils.core_helpers import attribute_or_none @@ -1787,7 +1793,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, - **_per_call_custom_pricing(litellm_params), + **litellm_params.custom_pricing_overrides(), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -2013,6 +2019,53 @@ def delete_responses( ) +def _requested_retrieve_stream_query( + stream: bool | None, + starting_after: int | None, + extra_query: Mapping[str, object] | None, + custom_llm_provider: str | None, + drop_malformed: bool, +) -> RetrieveStreamQuery: + explicit: Final = { + key: value for key, value in (("stream", stream), ("starting_after", starting_after)) if value is not None + } + requested: Final = {**explicit, **(extra_query or {})} + parsed: Final = parse_retrieve_stream_query( + requested.get("stream"), requested.get("starting_after"), drop_malformed=drop_malformed + ) + if isinstance(parsed, InvalidRetrieveStreamQuery): + raise litellm.BadRequestError( + message=parsed.message, + model=None, + llm_provider=custom_llm_provider, + body={"message": parsed.message, "type": "invalid_request_error", "param": parsed.param}, + ) + return parsed + + +def _supported_retrieve_stream_query( + requested: RetrieveStreamQuery, + responses_api_provider_config: BaseResponsesAPIConfig, + custom_llm_provider: str, + drop_params: object, +) -> RetrieveStreamQuery: + if not requested.stream: + return RetrieveStreamQuery(stream=False, starting_after=None) + if responses_api_provider_config.supports_streamed_retrieval(): + return requested + if drops_unsupported_params(drop_params): + return RetrieveStreamQuery(stream=False, starting_after=None) + raise UnsupportedParamsError( + status_code=400, + llm_provider=custom_llm_provider, + message=( + f"{custom_llm_provider} does not support streaming a stored response (stream=true on " + "GET /v1/responses/{response_id}). To drop it, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + ) + + @client async def aget_responses( response_id: str, @@ -2024,8 +2077,10 @@ async def aget_responses( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + stream: bool | None = None, + starting_after: int | None = None, **kwargs, -) -> ResponsesAPIResponse: +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: """ Async: Fetch a response by its ID. @@ -2058,6 +2113,8 @@ async def aget_responses( extra_query=extra_query, extra_body=extra_body, timeout=timeout, + stream=stream, + starting_after=starting_after, **kwargs, ) @@ -2071,7 +2128,7 @@ async def aget_responses( response = init_response # Update the responses_api_response_id with the model_id - if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime + if not isinstance(response, ResponsesAPIResponse): return response return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, @@ -2101,8 +2158,14 @@ def get_responses( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + stream: bool | None = None, + starting_after: int | None = None, **kwargs, -) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]: +) -> ( + ResponsesAPIResponse + | SyncResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator] +): """ Fetch a response by its ID. @@ -2146,6 +2209,20 @@ def get_responses( if responses_api_provider_config is None: raise ValueError(f"GET responses is not supported for {custom_llm_provider}") + stream_query: Final = _supported_retrieve_stream_query( + requested=_requested_retrieve_stream_query( + stream=stream, + starting_after=starting_after, + extra_query=extra_query, + custom_llm_provider=custom_llm_provider, + drop_malformed=drops_unsupported_params(litellm_params.drop_params), + ), + responses_api_provider_config=responses_api_provider_config, + custom_llm_provider=custom_llm_provider, + drop_params=litellm_params.drop_params, + ) + litellm_metadata: Final = _STR_DICT.validate_python(kwargs.get("litellm_metadata") or {}) + local_vars.update(kwargs) # Pre Call logging @@ -2169,7 +2246,13 @@ def get_responses( litellm_params=litellm_params, logging_obj=litellm_logging_obj, extra_headers=extra_headers, + extra_query={ + key: value for key, value in (extra_query or {}).items() if key in RETRIEVE_INCLUDE_QUERY_KEYS + }, extra_body=extra_body, + stream=stream_query.stream, + starting_after=stream_query.starting_after, + litellm_metadata=litellm_metadata, timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), @@ -2180,7 +2263,7 @@ def get_responses( if isinstance(response, ResponsesAPIResponse): response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/responses/retrieve_stream_query.py b/litellm/responses/retrieve_stream_query.py new file mode 100644 index 00000000000..086aa39efc6 --- /dev/null +++ b/litellm/responses/retrieve_stream_query.py @@ -0,0 +1,83 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, Literal + +from pydantic import TypeAdapter + +RETRIEVE_STREAM_QUERY_KEYS: Final = frozenset({"stream", "starting_after"}) +RETRIEVE_INCLUDE_QUERY_KEYS: Final = frozenset({"include", "include[]", "include_obfuscation"}) +_QUERY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_MAX_STARTING_AFTER_DIGITS: Final = len(str(2**63 - 1)) + + +@dataclass(frozen=True, slots=True) +class RetrieveStreamQuery: + stream: bool + starting_after: int | None + + +@dataclass(frozen=True, slots=True) +class InvalidRetrieveStreamQuery: + param: Literal["stream", "starting_after"] + message: str + + +_NO_STREAM: Final = RetrieveStreamQuery(stream=False, starting_after=None) +_FULL_REPLAY: Final = RetrieveStreamQuery(stream=True, starting_after=None) + + +def _parsed_stream(value: object) -> bool | InvalidRetrieveStreamQuery: + match value: + case None: + return False + case bool(): + return value + case str() if value.lower() in ("true", "false"): + return value.lower() == "true" + case _: + return InvalidRetrieveStreamQuery("stream", f"Invalid 'stream': expected true or false, got {value!r}") + + +def _parsed_starting_after(value: object) -> int | None | InvalidRetrieveStreamQuery: + match value: + case None: + return None + case int() if not isinstance(value, bool) and value >= 0: + return value + case str() if value.isascii() and value.isdigit() and len(value) <= _MAX_STARTING_AFTER_DIGITS: + return int(value) + case _: + return InvalidRetrieveStreamQuery( + "starting_after", f"Invalid 'starting_after': expected an integer >= 0, got {value!r}" + ) + + +def parse_retrieve_stream_query( + stream: object, starting_after: object, drop_malformed: bool +) -> RetrieveStreamQuery | InvalidRetrieveStreamQuery: + parsed_stream: Final = _parsed_stream(stream) + if isinstance(parsed_stream, InvalidRetrieveStreamQuery): + return _NO_STREAM if drop_malformed else parsed_stream + if not parsed_stream: + return _NO_STREAM + parsed_starting_after: Final = _parsed_starting_after(starting_after) + if isinstance(parsed_starting_after, InvalidRetrieveStreamQuery): + return _FULL_REPLAY if drop_malformed else parsed_starting_after + return RetrieveStreamQuery(stream=True, starting_after=parsed_starting_after) + + +def requests_streamed_retrieval(request: Mapping[str, object]) -> bool: + extra_query: Final = request.get("extra_query") + return ( + "response_id" in request + and isinstance(extra_query, Mapping) + and _parsed_stream(_QUERY_ADAPTER.validate_python(extra_query).get("stream")) is True + ) + + +def without_retrieve_stream_defaults( + deployment_params: Mapping[str, object], request: Mapping[str, object] +) -> Mapping[str, object]: + if "response_id" not in request: + return deployment_params + return {key: value for key, value in deployment_params.items() if key not in RETRIEVE_STREAM_QUERY_KEYS} diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6c57bfd2a3f..8959c32fff2 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -376,6 +376,7 @@ class BaseResponsesAPIStreamingIterator: _pending_logging_tasks: tuple[asyncio.Task[object], ...] = () _started_tool_call: bool = False _earlier_rounds_spend: BilledRoundsSpend | None = None + _replays_stored_stream: bool = False def __init__( self, @@ -574,7 +575,7 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): _response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None) - _estimate_wanted: Final[bool] = _chunk_type in ( + _estimate_wanted: Final[bool] = not self._replays_stored_stream and _chunk_type in ( openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, ) @@ -995,6 +996,7 @@ class BaseResponsesAPIStreamingIterator: generated_text: Final = self._generated_content + self._generated_tool_arguments if ( not generated_text + or self._replays_stored_stream or requests_background(self.request_data) or is_background_response(self._created_response) or self._completed_response_logged @@ -1110,7 +1112,7 @@ class BaseResponsesAPIStreamingIterator: """ created: Final = self._created_response generated: Final = self._generated_content + self._generated_tool_arguments - if created is None or not generated or self._billed_without_disconnect(): + if self._replays_stored_stream or created is None or not generated or self._billed_without_disconnect(): return None usage: Final = _estimate_usage_safely( self.model or "", @@ -1162,6 +1164,8 @@ class BaseResponsesAPIStreamingIterator: def _raise_if_ended_without_terminal_event(self) -> None: if self.completed_response is not None: return + if self._replays_stored_stream and not self._yielded_first_chunk: + return error: Final = self._stream_ended_early_error() self._handle_failure(error) if self._output_started: @@ -1209,6 +1213,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): custom_llm_provider: str | None = None, request_data: dict[str, object] | None = None, call_type: str | None = None, + replays_stored_stream: bool = False, ): super().__init__( response, @@ -1220,6 +1225,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): request_data, call_type, ) + self._replays_stored_stream = replays_stored_stream self.stream_iterator = SSEDecoder().aiter_bytes(response.aiter_bytes()) def __aiter__(self): @@ -1232,6 +1238,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): await self._await_pending_logging() raise + async def aclose(self) -> None: + await self.response.aclose() + async def _next_event(self) -> ResponsesAPIStreamingResponse: try: self._check_max_streaming_duration() @@ -1304,6 +1313,7 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): custom_llm_provider: str | None = None, request_data: dict[str, object] | None = None, call_type: str | None = None, + replays_stored_stream: bool = False, ): super().__init__( response, @@ -1315,11 +1325,15 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): request_data, call_type, ) + self._replays_stored_stream = replays_stored_stream self.stream_iterator = SSEDecoder().iter_bytes(response.iter_bytes()) def __iter__(self): return self + def close(self) -> None: + self.response.close() + def __next__(self): try: self._check_max_streaming_duration() diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..b0db0516658 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -131,14 +131,14 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts @@ -287,7 +287,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. diff --git a/litellm/router.py b/litellm/router.py index b63130c3313..ff31f48c49e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -75,7 +75,7 @@ from typing_extensions import assert_never, overload import litellm import litellm.litellm_core_utils.exception_mapping_utils from litellm import get_secret_str -from litellm._internal_context import service_target, with_service_target +from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid from litellm.caching.caching import ( @@ -162,7 +162,11 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_structure, ) -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import ( + offload_token_count, + served_openai_models, + single_encoding_openai_model, +) from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception from litellm.llms.base_llm.passthrough.transformation import replace_path_segment @@ -180,6 +184,7 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.responses.bridge_check import keep_registered_mode +from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_strategy.auto_router.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, STRATEGY_ROUTER_PARAM_FIELDS, @@ -719,6 +724,29 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None: + from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request + + configured: Final = _configured_model_info(deployment).get("supports_search_tool") + if isinstance(configured, bool): + return configured + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return None + try: + resolved_model, resolved_provider = resolve_model_and_provider_for_metadata( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=resolved_model, provider=LlmProviders(resolved_provider) + ) + except (litellm.BadRequestError, ValueError): + return None + return True if chat_config_builds_anthropic_request(chat_config) else None + + def _every_deployment_refused( deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...] ) -> bool: @@ -726,7 +754,7 @@ def _every_deployment_refused( def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: - if silent_model is None: + if silent_model is None or in_mirrored_call(): return () try: targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) @@ -1533,6 +1561,7 @@ class Router: self._cached_deployment_tracks_usage = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( self._deployment_tracks_usage ) + self.served_openai_models = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(self._served_openai_models) self._discovered_model_info_cache: InMemoryCache = InMemoryCache( max_size_in_memory=max(len(model_list or ()), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, @@ -3343,16 +3372,17 @@ class Router: remaining_kwargs: Final = MappingProxyType( {key: value for key, value in silent_kwargs.items() if key != "stream"} ) - response: Final = await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - stream=bool(silent_kwargs.get("stream", False)), - **remaining_kwargs, - ) - if not isinstance(response, CustomStreamWrapper): - return - async for _ in response: - pass + with mirrored_call(): + response: Final = await self.acompletion( + model=silent_model, + messages=cast(list[AllMessageValues], messages), + stream=bool(silent_kwargs.get("stream", False)), + **remaining_kwargs, + ) + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): """ @@ -4832,7 +4862,7 @@ class Router: Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name """ # 1) copy your defaults and pull out metadata - defaults: Final = self.default_litellm_params.copy() + defaults: Final = dict(without_retrieve_stream_defaults(self.default_litellm_params, kwargs)) metadata_defaults: Final = defaults.pop("metadata", {}) or {} # 2) add any non-metadata defaults that aren't already in kwargs @@ -5073,12 +5103,13 @@ class Router: router_timeout=_router_timeout, router_stream_timeout=_router_stream_timeout, ) + timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else kwargs kwargs.update( CompletionTimeout.stream_request_kwargs( - passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=kwargs) + passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request) ) - if splits_stream_timeout and kwargs.get("stream") - else {"timeout": passthrough_timeout(kwargs=kwargs)} + if splits_stream_timeout and timeout_request.get("stream") + else {"timeout": passthrough_timeout(kwargs=timeout_request)} ) elif splits_stream_timeout: request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) @@ -6412,7 +6443,7 @@ class Router: generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "") response_kwargs: Final = { - **data, + **without_retrieve_stream_defaults(data, kwargs), "caching": self.cache_responses, **kwargs, "model": model_name, @@ -11890,6 +11921,9 @@ class Router: indices: Final = self.model_name_to_deployment_indices.get(model_name) return None if indices is None else tuple(self.model_list[idx] for idx in indices) + def _served_openai_models(self, model_name: str) -> tuple[str | None, ...]: + return served_openai_models(self.deployments_named(model_name) or ()) + def team_public_model_names_of(self, team_id: str) -> frozenset[str]: """The public names `team_id` serves its own deployments under.""" with self._deployment_index_lock: @@ -12226,6 +12260,15 @@ class Router: for deployment in self._routable_deployments(model_name, team_id) ) + def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: + """ + Whether each deployment a request from team_id to model_name can route + to (see _routable_deployments) answers Codex's client `tool_search`: + its model_info.supports_search_tool when that is a bool, else True when + its route builds an Anthropic Messages request, else None (unknown). + """ + return tuple(_search_tool_support(deployment) for deployment in self._routable_deployments(model_name, team_id)) + def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: """ Return the `litellm_params.model` of the first deployment a request @@ -14044,6 +14087,7 @@ class Router: self._ptu_share_teams.cache_clear() self._cached_team_ptu_ceiling.cache_clear() self._cached_deployment_tracks_usage.cache_clear() + self.served_openai_models.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None @@ -14303,6 +14347,7 @@ class Router: messages: Sequence[Mapping[str, object]] | None, input: str | list[object] | None, request_kwargs: Mapping[str, object] | None = None, + tokenizer_model: str = "", ) -> int: """ Count input tokens for context-window pre-call checks. @@ -14333,7 +14378,7 @@ class Router: system_message: Final = anthropic_system_to_openai_message(extras.get("system")) if messages is not None: counted_messages: Final = (system_message, *messages) if system_message is not None else messages - return litellm.token_counter(messages=counted_messages, tools=tools) + return litellm.token_counter(model=tokenizer_model, messages=counted_messages, tools=tools) if input is not None: from openai.types.responses.response_create_params import ResponseInputParam @@ -14348,6 +14393,7 @@ class Router: only_since_compaction=True, ) return litellm.token_counter( + model=tokenizer_model, messages=cast(list, input_messages), # cast-ok: transformed chat messages tools=tools, ) @@ -14404,6 +14450,10 @@ class Router: messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter input=cast(str | list | None, input), # cast-ok: forwarded to the sync counter request_kwargs=request_kwargs, + tokenizer_model=single_encoding_openai_model( + self.served_openai_models(model) or served_openai_models(healthy_deployments) + ) + or "", ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( @@ -14418,6 +14468,8 @@ class Router: request_kwargs: Mapping[str, object] | None, input_token_count: int | None, count_inline: bool, + model: str, + deployments: Sequence[Mapping[str, object]], ) -> int | None: if input_token_count is not None: return input_token_count @@ -14428,6 +14480,10 @@ class Router: messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter input=cast(str | list[object] | None, input), # cast-ok: forwarded to the sync counter request_kwargs=request_kwargs, + tokenizer_model=single_encoding_openai_model( + self.served_openai_models(model) or served_openai_models(deployments) + ) + or "", ) except Exception as e: verbose_router_logger.error( @@ -14622,6 +14678,8 @@ class Router: count_inline=has_countable_input and not skip_inline_token_count and any(limit is not None for limit in deployment_max_input_tokens), + model=model, + deployments=_returned_deployments, ) request_settings: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else {} requested_params: Final[frozenset[str]] = ( diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 2e350a2f87b..0b3d48c9e1d 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -22,7 +22,7 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.router import RouterCacheEnum, RouterErrors @@ -127,6 +127,12 @@ def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | return itpm_key, otpm_key +def _tokenizer_model(deployment: object) -> str: + params: Final = deployment.get("litellm_params") if is_str_keyed_mapping(deployment) else None + served: Final = params.get("model") if is_str_keyed_mapping(params) else None + return deployment_openai_model(deployment) or (served if isinstance(served, str) else "") + + def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "") -> int: if not request_kwargs: return 0 @@ -427,7 +433,7 @@ def io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = (deployment.get("litellm_params") or {}).get("model") or "" + _model: Final = _tokenizer_model(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) @@ -491,7 +497,7 @@ async def async_io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = (deployment.get("litellm_params") or {}).get("model") or "" + _model: Final = _tokenizer_model(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..3c3fad93579 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,6 +426,27 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, @@ -434,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74cd28161a4..fe52f8416d0 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -96,6 +96,8 @@ class Feature(str, Enum): """litellm.include_cost_in_streaming_usage, where Python adds cost to streamed usage.""" THINKING_SIGNATURE_RETRY = "thinking_signature_retry" """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" + TOOL_CUSTOM_FIELD = "tool_custom_field" + """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -176,6 +178,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.CACHE_CONTROL_INJECTION, Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, + Feature.TOOL_CUSTOM_FIELD, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 5f75858d350..52917126b3b 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -43,6 +43,11 @@ def thinking_signature_retry(request: NativeCall) -> bool: return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ()) +def tool_custom_field(request: NativeCall) -> bool: + tools: Final = optional_sequence(request.resolved.get("tools")) or () + return any("custom" in (optional_mapping(tool) or {}) for tool in tools) + + def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index aaa0d62eea7..d0b2850bb5b 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -63,6 +63,8 @@ def _assistant_tool_call(tool_call: ChatCompletionMessageToolCall) -> ChatComple def _assistant_message( message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], replays_provider_specific_fields: bool ) -> ChatCompletionAssistantMessage: + from litellm.completion_extras.litellm_responses_transformation.history import retained_replay_fields + return cast( # cast-ok: dict literal with the optional message fields spread in only when set "ChatCompletionAssistantMessage", { @@ -78,7 +80,7 @@ def _assistant_message( "provider_specific_fields", getattr(message, "provider_specific_fields", None) if replays_provider_specific_fields - else None, + else retained_replay_fields(getattr(message, "provider_specific_fields", None)), ), ) if value is not None diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 13542f22f88..77e75dea030 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -97,9 +97,9 @@ class AgenticLoopRequestPatch(LiteLLMBaseModel): model: str | None = None messages: list[dict[str, Any]] | None = None - tools: list[dict[str, Any]] | None = None + tools: list[dict[str, object]] | None = None max_tokens: int | None = None - optional_params: dict[str, Any] = Field(default_factory=dict) + optional_params: dict[str, object] = Field(default_factory=dict) kwargs: dict[str, Any] = Field(default_factory=dict) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index f4fcabf53ed..f2042cb7967 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -5,7 +5,7 @@ API Reference: https://docs.datadoghq.com/llm_observability/setup/api/?tab=examp """ from collections.abc import Sequence -from typing import Any, Literal +from typing import Literal from typing_extensions import ReadOnly, TypedDict @@ -16,7 +16,7 @@ class ToolCall(TypedDict, total=False): """A tool call on a message, as LLM Obs names its fields.""" name: ReadOnly[str] - arguments: ReadOnly[dict[str, Any] | str] # parsed object, or the raw string when it will not parse to one + arguments: ReadOnly[dict[str, object] | str] # parsed object, or the raw string when it will not parse to one tool_id: ReadOnly[str] type: ReadOnly[str] @@ -35,7 +35,7 @@ class ToolDefinition(TypedDict, total=False): name: ReadOnly[str] description: ReadOnly[str] - schema: ReadOnly[dict[str, Any]] + schema: ReadOnly[dict[str, object]] class Message(TypedDict, total=False): @@ -49,13 +49,15 @@ class Message(TypedDict, total=False): class InputMeta(TypedDict): - messages: Sequence[ - Message | dict[str, Any] # changed to fit with tool calls + messages: ReadOnly[ + Sequence[ + Message | dict[str, object] # changed to fit with tool calls + ] ] # Relevant Issue: https://github.com/BerriAI/litellm/issues/9494 class OutputMeta(TypedDict): - messages: Sequence[Any] + messages: ReadOnly[Sequence[object]] class DDLLMObsError(TypedDict, total=False): @@ -71,7 +73,7 @@ class Meta(TypedDict, total=False): kind: Literal["llm", "tool", "task", "embedding", "retrieval"] input: InputMeta # The span's input information. output: OutputMeta # The span's output information. - metadata: dict[str, Any] + metadata: ReadOnly[dict[str, object]] error: DDLLMObsError | None # Error information on the span tool_definitions: ReadOnly[Sequence[ToolDefinition]] # The tools offered to the model on this request diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index de83e6ea66e..50c4a2340a3 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -832,6 +832,7 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" +ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01" ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01" ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8ef280c52cf..a54d66c4beb 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1604,7 +1604,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): @field_validator("reasoning", mode="before") @classmethod - def validate_reasoning_to_dict(cls, value: Any) -> dict[str, Any] | None: + def validate_reasoning_to_dict(cls, value: Any) -> builtins.object: """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" if value is None: return None diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index f1176f777c7..a938f6a7b1c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from enum import Enum -from typing import Any, Final, Literal, Protocol +from typing import Final, Literal, Protocol from typing_extensions import ( ReadOnly, @@ -140,7 +140,7 @@ class Schema(TypedDict, total=False): title: str description: str nullable: bool - default: Any + default: ReadOnly[object] items: "Schema" minItems: str maxItems: str @@ -155,7 +155,7 @@ class Schema(TypedDict, total=False): minLength: str maxLength: str pattern: str - example: Any + example: ReadOnly[object] anyOf: list["Schema"] @@ -376,7 +376,7 @@ class RequestBody(TypedDict, total=False): class CachedContentRequestBody(TypedDict, total=False): contents: Required[list[ContentType]] system_instruction: SystemInstructions - tools: Tools + tools: Tools | None toolConfig: ToolConfig model: Required[str] # Format: models/{model} ttl: str # ending in 's' - Example: "3.5s". @@ -794,9 +794,9 @@ class VertexVideoGenerationResponse(TypedDict, total=False): name: str done: bool - response: dict[str, Any] - metadata: dict[str, Any] - error: dict[str, Any] + response: ReadOnly[dict[str, object]] + metadata: ReadOnly[dict[str, object]] + error: ReadOnly[dict[str, object]] VERTEX_CREDENTIALS_TYPES = str | dict[str, str] diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index d396c79b844..ca6ada0d1ec 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -15,4 +15,5 @@ class UiDiscoveryEndpoints(LiteLLMBaseModel): is_control_plane: bool = False mcp_stdio_enabled: bool = False use_si_terminology: bool = False + claude_code_marketplace_auth_required: bool = False workers: list[WorkerRegistryEntry] = Field(default=[]) diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index d38e54986b9..4e4983c53f0 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -48,7 +48,7 @@ class GenericGuardrailAPIMetadata(TypedDict, total=False): class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel): """Optional parameters for the Generic Guardrail API""" - additional_provider_specific_params: dict[str, Any] | None = Field( + additional_provider_specific_params: dict[str, object] | None = Field( default=None, description="Additional provider-specific parameters to send with the guardrail request", ) @@ -161,7 +161,7 @@ class GenericGuardrailAPIRequest(LiteLLMBaseModel): default=None, description="LiteLLM library version running this proxy.", ) - additional_provider_specific_params: dict[str, Any] | None = None + additional_provider_specific_params: dict[str, object] | None = None tool_calls: list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] | None = None model: str | None = None # the model being used for the LLM call diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py index b58d1bff8a2..b04be71dc9c 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py @@ -1,7 +1,5 @@ -from typing import Any - from pydantic import Field -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -14,41 +12,41 @@ from .base import IBMGuardrailsBaseConfigModel class IBMDetectorRequestBodyDetectorServer(TypedDict): """Request body for calling IBM Detector Server directly""" - contents: list[str] - detector_params: dict[str, Any] + contents: ReadOnly[list[str]] + detector_params: ReadOnly[dict[str, object]] class IBMDetectorRequestBodyOrchestrator(TypedDict): """Request body for calling IBM Detector via FMS Guardrails Orchestrator""" - content: str - detectors: dict[str, dict[str, Any]] + content: ReadOnly[str] + detectors: ReadOnly[dict[str, dict[str, object]]] class IBMDetectorDetection(TypedDict, total=False): """Individual detection from IBM Detector""" - start: int - end: int - text: str - detection: str - detection_type: str - score: float - evidences: list[Any] - metadata: dict[str, Any] - detector_id: str | None # Only present in orchestrator response + start: ReadOnly[int] + end: ReadOnly[int] + text: ReadOnly[str] + detection: ReadOnly[str] + detection_type: ReadOnly[str] + score: ReadOnly[float] + evidences: ReadOnly[list[object]] + metadata: ReadOnly[dict[str, object]] + detector_id: ReadOnly[str | None] # Only present in orchestrator response class IBMDetectorResponseDetectorServer(TypedDict): """Response from IBM Detector Server (returns list of lists)""" - detections: list[list[IBMDetectorDetection]] + detections: ReadOnly[list[list[IBMDetectorDetection]]] class IBMDetectorResponseOrchestrator(TypedDict): """Response from IBM FMS Guardrails Orchestrator""" - detections: list[IBMDetectorDetection] + detections: ReadOnly[list[IBMDetectorDetection]] # Pydantic Config Models @@ -57,12 +55,12 @@ class IBMDetectorResponseOrchestrator(TypedDict): class IBMDetectorOptionalParams(LiteLLMBaseModel): """Optional parameters for IBM Detector guardrail""" - detector_params: dict[str, Any] | None = Field( + detector_params: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of arguments to pass to the detector.", ) - extra_headers: dict[str, Any] | None = Field( + extra_headers: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of extra headers to pass to the detector.", ) diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index f4348391710..d4f41b538e6 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict @@ -50,14 +50,14 @@ class SuccessfulKeyUpdate(LiteLLMBaseModel): """Successfully updated key with its updated information""" key: str - key_info: dict[str, Any] + key_info: dict[str, object] class FailedKeyUpdate(LiteLLMBaseModel): """Failed key update with reason""" key: str - key_info: dict[str, Any] | None = None + key_info: dict[str, object] | None = None failed_reason: str @@ -78,14 +78,14 @@ class KeyUpdateFields(LiteLLMBaseModel): max_budget: float | None = None budget_id: str | None = None budget_duration: str | None = None - budget_limits: list[Any] | None = None - model_max_budget: dict[str, Any] | None = None + budget_limits: list[object] | None = None + model_max_budget: dict[str, object] | None = None # Rate limits tpm_limit: int | None = None rpm_limit: int | None = None - model_tpm_limit: dict[str, Any] | None = None - model_rpm_limit: dict[str, Any] | None = None + model_tpm_limit: dict[str, object] | None = None + model_rpm_limit: dict[str, object] | None = None max_parallel_requests: int | None = None rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None @@ -99,7 +99,7 @@ class KeyUpdateFields(LiteLLMBaseModel): # Operational metadata tags: list[str] | None = None - metadata: dict[str, Any] | None = None + metadata: dict[str, object] | None = None @model_validator(mode="after") def validate_temp_budget(self) -> "KeyUpdateFields": diff --git a/litellm/types/router.py b/litellm/types/router.py index 60c9e596588..db8be9e91a3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -67,6 +67,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6f15fd92790..727857bc118 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -265,10 +265,10 @@ class AgenticLoopParams(TypedDict, total=False): """ model: str - """The model string with provider prefix (e.g., 'bedrock/invoke/...')""" + """The model as the caller requested it, usually with its provider prefix (e.g., 'bedrock/invoke/...')""" custom_llm_provider: str - """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + """The LLM provider name (e.g., 'bedrock', 'anthropic'), when the recorder resolved one""" class OffPeakWindow(TypedDict, total=False): @@ -3397,6 +3397,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): @@ -4153,6 +4154,18 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): """ return {k: v for k, v in model_info.items() if k not in cls.model_fields} + def custom_pricing_overrides(self) -> Mapping[str, object]: + return MappingProxyType( + self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + def per_second_pricing(self) -> Mapping[str, float]: + return MappingProxyType( + self.model_dump( + include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True + ) + ) + DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) @@ -4447,6 +4460,7 @@ class LlmProviders(str, Enum): JINA_AI = "jina_ai" XAI = "xai" ZAI = "zai" + BIGMODEL = "bigmodel" CUSTOM_OPENAI = "custom_openai" TEXT_COMPLETION_OPENAI = "text-completion-openai" COHERE = "cohere" @@ -4621,6 +4635,7 @@ class LlmProviders(str, Enum): CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + KYMA = "kyma" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 72c37c3760e..a656d8ebca3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -29,8 +29,8 @@ import threading import time import traceback from contextlib import nullcontext -from dataclasses import dataclass, field -from functools import lru_cache, wraps +from dataclasses import dataclass, field, replace +from functools import lru_cache, partial, wraps from importlib import resources from inspect import iscoroutine from io import StringIO @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -2032,16 +2033,15 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() - if _is_streaming_request( - kwargs=kwargs, - call_type=call_type, - ): + if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + logging_obj.stream = True + logging_obj.model_call_details["stream"] = True # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2578,6 +2578,8 @@ _NEVER_STREAMING_CALL_TYPES: Final = frozenset( } ) +_RESULT_DECIDES_STREAMING_CALL_TYPES: Final = frozenset({"get_responses", "aget_responses"}) + def _is_streaming_request( kwargs: dict[str, object], @@ -2589,7 +2591,7 @@ def _is_streaming_request( - if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages) - if call_type is generate_content_stream or agenerate_content_stream (litellm google genai) """ - if call_type in _NEVER_STREAMING_CALL_TYPES: + if call_type in _NEVER_STREAMING_CALL_TYPES or call_type in _RESULT_DECIDES_STREAMING_CALL_TYPES: return False if "stream" in kwargs and kwargs["stream"] is True: return True @@ -2682,9 +2684,99 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: return None +@dataclass(frozen=True, slots=True) +class _HuggingFaceTokenizerLoaded: + tokenizer: HuggingFace + + @dataclass(frozen=True, slots=True) class _HuggingFaceTokenizerLoadFailure: reason: str + retry_at: float + retry_pid: int | None = None + + +_HuggingFaceTokenizerKey: TypeAlias = tuple[HuggingFaceTokenizerKind, Decision] +_HuggingFaceTokenizerLoadOutcome: TypeAlias = _HuggingFaceTokenizerLoaded | _HuggingFaceTokenizerLoadFailure + + +def _start_in_thread_joined_at_exit(work: Callable[[], None]) -> None: + threading.Thread(target=work, name="litellm-huggingface-tokenizer-retry", daemon=False).start() + + +@dataclass(slots=True) +class _HuggingFaceTokenizerLoadOutcomes: + clock: Callable[[], float] = time.monotonic + start_retry: Callable[[Callable[[], None]], None] = _start_in_thread_joined_at_exit + pid: Callable[[], int] = os.getpid + lock: threading.Lock = field(default_factory=threading.Lock) + by_tokenizer: Mapping[_HuggingFaceTokenizerKey, _HuggingFaceTokenizerLoadOutcome] = field( + default_factory=lambda: MappingProxyType({}) + ) + + def known( + self, key: _HuggingFaceTokenizerKey, retry_when_due: Callable[[], None] + ) -> _HuggingFaceTokenizerLoadOutcome | None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if ( + not isinstance(known, _HuggingFaceTokenizerLoadFailure) + or known.retry_pid == self.pid() + or self.clock() < known.retry_at + ): + return known + retrying: Final = replace(known, retry_pid=self.pid()) + self._remember(key, retrying) + try: + self.start_retry(retry_when_due) + except RuntimeError as error: + return self.remember_failed_retry(key, str(error)) + return retrying + + def remember_loaded(self, key: _HuggingFaceTokenizerKey, tokenizer: HuggingFace) -> _HuggingFaceTokenizerLoaded: + loaded: Final = _HuggingFaceTokenizerLoaded(tokenizer) + with self.lock: + self._remember(key, loaded) + return loaded + + def remember_first_failure(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if known is not None: + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def remember_failed_retry(self, key: _HuggingFaceTokenizerKey, reason: str) -> _HuggingFaceTokenizerLoadOutcome: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoaded): + return known + failure: Final = self._failure(reason) + self._remember(key, failure) + return failure + + def end_retry(self, key: _HuggingFaceTokenizerKey) -> None: + with self.lock: + known: Final = self.by_tokenizer.get(key) + if isinstance(known, _HuggingFaceTokenizerLoadFailure) and known.retry_pid is not None: + self._remember(key, self._failure(known.reason)) + + def flush_cache(self) -> None: + with self.lock: + self.by_tokenizer = MappingProxyType({}) + + def _failure(self, reason: str) -> _HuggingFaceTokenizerLoadFailure: + return _HuggingFaceTokenizerLoadFailure( + reason=reason, retry_at=self.clock() + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS + ) + + def _remember(self, key: _HuggingFaceTokenizerKey, outcome: _HuggingFaceTokenizerLoadOutcome) -> None: + self.by_tokenizer = MappingProxyType({**self.by_tokenizer, key: outcome}) + + +_HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES: Final = _HuggingFaceTokenizerLoadOutcomes() def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: @@ -2696,30 +2788,28 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: ) -def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: +def _return_huggingface_tokenizer( + model: str, outcomes: _HuggingFaceTokenizerLoadOutcomes = _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES +) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): return None - backend: Final = _huggingface_tokenizer_backend() - key: Final = _huggingface_tokenizer_failure_key(kind, backend) + key: Final = (kind, _huggingface_tokenizer_backend()) with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): - cached_failure: Final = _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.get_cache(key) - if isinstance(cached_failure, _HuggingFaceTokenizerLoadFailure): - return cached_failure - tokenizer: Final = _load_huggingface_tokenizer_recording_failure(kind, backend, key) - if isinstance(tokenizer, _HuggingFaceTokenizerLoadFailure): - _warn_huggingface_tokenizer_fallback(model, tokenizer.reason) - return tokenizer - return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + known: Final = outcomes.known(key, partial(_retry_huggingface_tokenizer_load, model, key, outcomes)) + outcome: Final = ( + known + if known is not None + else _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_first_failure) + ) + match outcome: + case _HuggingFaceTokenizerLoaded(tokenizer=tokenizer): + return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + case _HuggingFaceTokenizerLoadFailure(): + return outcome -def _huggingface_tokenizer_failure_key(kind: HuggingFaceTokenizerKind, backend: Decision) -> str: - return f"{kind}:{backend.name}" - - -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: - """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> HuggingFace: match kind: case "cohere": return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") @@ -2739,19 +2829,34 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio assert_never(kind) -def _load_huggingface_tokenizer_recording_failure( - kind: HuggingFaceTokenizerKind, backend: Decision, key: str -) -> HuggingFace | _HuggingFaceTokenizerLoadFailure: +def _load_huggingface_tokenizer_remembering_outcome( + model: str, + key: _HuggingFaceTokenizerKey, + outcomes: _HuggingFaceTokenizerLoadOutcomes, + remember_failure: Callable[[_HuggingFaceTokenizerKey, str], _HuggingFaceTokenizerLoadOutcome], +) -> _HuggingFaceTokenizerLoadOutcome: + kind, _ = key try: - return _load_huggingface_tokenizer(kind, backend) + tokenizer: Final = _load_huggingface_tokenizer(kind) except Exception as error: from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): raise - failure: Final = _HuggingFaceTokenizerLoadFailure(reason=str(error)) - _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.set_cache(key, failure, ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS) - return failure + outcome: Final = remember_failure(key, str(error)) + if isinstance(outcome, _HuggingFaceTokenizerLoadFailure): + _warn_huggingface_tokenizer_fallback(model, outcome.reason) + return outcome + return outcomes.remember_loaded(key, tokenizer) + + +def _retry_huggingface_tokenizer_load( + model: str, key: _HuggingFaceTokenizerKey, outcomes: _HuggingFaceTokenizerLoadOutcomes +) -> None: + try: + _load_huggingface_tokenizer_remembering_outcome(model, key, outcomes, outcomes.remember_failed_retry) + finally: + outcomes.end_retry(key) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -3397,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. @@ -3711,7 +3852,7 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMa return None -def is_generalized_model_info(model_info: ModelInfo) -> bool: +def is_generalized_model_info(model_info: ModelInfoBase) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. Detected as the resolved key missing ``litellm.model_cost`` while matching a @@ -3727,10 +3868,12 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 +_ModelInfoT: Final = TypeVar("_ModelInfoT", bound=ModelInfoBase) -def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + +def _lookup_mapped_model_info(lookup: Callable[[], _ModelInfoT]) -> _ModelInfoT | ModelInfoLookupMiss: try: - info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + info: Final = lookup() except ModelNotMappedError: return ModelInfoLookupMiss(tripped=False) except Exception: @@ -3738,6 +3881,16 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + return _lookup_mapped_model_info(lambda: get_model_info(model=model, custom_llm_provider=custom_llm_provider)) + + +def lookup_catalog_model_info(model: str, custom_llm_provider: str | None) -> ModelInfoBase | ModelInfoLookupMiss: + return _lookup_mapped_model_info( + lambda: get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True) + ) + + def _get_builtin_model_info_for_registration( model: str, custom_llm_provider: str | None ) -> ModelInfo | ModelInfoLookupMiss: @@ -5003,6 +5156,46 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5092,19 +5285,30 @@ def get_optional_params( verbose_logger.debug( "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) ) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): + verbose_logger.warning( + _dropped_reasoning_warning(base_model or model, custom_llm_provider), + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + ) else: raise UnsupportedParamsError( status_code=500, @@ -5674,6 +5878,7 @@ def add_provider_specific_params_to_optional_params( "openai", "azure", "text-completion-openai", + "bigmodel", *litellm.openai_compatible_providers, ] or JSONProviderRegistry.exists(custom_llm_provider) if calls_through_openai_sdk: @@ -6619,6 +6824,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6652,7 +6859,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6676,7 +6883,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, @@ -8344,26 +8551,6 @@ def trim_messages( from litellm.caching.in_memory_cache import InMemoryCache -class _HuggingFaceTokenizerLoadFailureCache(Protocol): - def get_cache(self, key: str) -> object | None: ... - - def set_cache(self, key: str, value: object, ttl: float | None = None) -> None: ... - - def flush_cache(self) -> None: ... - - -_HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCache] = ( - cast( # cast-ok: cache methods are untyped - _HuggingFaceTokenizerLoadFailureCache, - InMemoryCache( - max_size_in_memory=16, - default_ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, - clock=lambda: time.monotonic(), - ), - ) -) - - class AvailableModelsCache(InMemoryCache): def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): super().__init__(ttl_seconds, max_size) @@ -9183,6 +9370,7 @@ class ProviderConfigManager: LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), + LlmProviders.BIGMODEL: (ProviderConfigManager._get_bigmodel_chat_config, False), LlmProviders.OPENCODE_GO: (ProviderConfigManager.get_opencode_go_config, True), LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False), @@ -9375,6 +9563,12 @@ class ProviderConfigManager: return get_bedrock_chat_config(model=model) + @staticmethod + def _get_bigmodel_chat_config() -> BaseConfig: + from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig + + return BigModelChatConfig() + @staticmethod def get_opencode_go_config(model: str) -> BaseConfig: from litellm.llms.opencode.chat.transformation import go_chat_config @@ -9696,6 +9890,10 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig + + return BigModelMessagesConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.messages.transformation import OpenCodeGoMessagesConfig @@ -9927,6 +10125,10 @@ class ProviderConfigManager: if provider is None: return None + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig + + return BigModelResponsesAPIConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.responses.transformation import OpenCodeGoResponsesConfig diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 23c6ccf9c0a..4fad4857c60 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28071,7 +28071,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, @@ -28345,7 +28345,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": true }, "fireworks_ai/qwen3p7-plus": { "cache_read_input_token_cost": 8e-08, @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -78339,6 +78341,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, @@ -78995,5 +80245,81 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/osv-scanner.toml b/osv-scanner.toml index 33e2f614edc..32a4f65fdf9 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -1,27 +1,102 @@ -[[IgnoredVulns]] -id = "GHSA-w8v5-vhqr-4h9v" -ignoreUntil = 2026-11-01 -reason = "diskcache has no fixed release published; remove this entry once one exists" - [[IgnoredVulns]] id = "GHSA-v4x8-3wr5-3wq8" ignoreUntil = 2026-11-20 -reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" +reason = "filelock ReadWriteLock advisory, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" [[IgnoredVulns]] -id = "GHSA-vfj7-8cjw-p6xm" -ignoreUntil = 2026-11-03 -reason = "braces has no fixed release published (3.0.3, 2024-05-21, is latest and last_affected); dev-only via knip > fast-glob > micromatch; remove this entry once one exists" +id = "GHSA-567q-5qvw-pf92" +ignoreUntil = 2026-11-20 +reason = "Windows-only local DoS (CVSS 2.5) in filelock soft locks; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" [[IgnoredVulns]] -id = "GHSA-6q48-6c7w-9gw2" -ignoreUntil = 2026-10-23 -reason = "certifi 2026.10.19 (published 2026-10-20) is inside the 3-day uv exclude-newer window until 2026-10-23 02:24 UTC; bump certifi and remove this entry once it clears" +id = "GHSA-9vhv-jxww-xfm5" +ignoreUntil = 2026-11-20 +reason = "Windows-only local DoS (CVSS 3.3) in filelock ReadWriteLock, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-6627-m3xw-93mp" +ignoreUntil = 2026-11-20 +reason = "filelock StrictSoftFileLock claim directories ignore the lock mode, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-69hj-f68j-2879" +ignoreUntil = 2026-11-20 +reason = "filelock UnixFileLock truncates and chmods a file another user renames or hard-links in at the lock path; litellm's CLI takes FileLock only inside ~/.litellm, which it creates owner-only, and huggingface_hub takes it in the Hugging Face cache, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting; the attack needs another local user who can rename or link files in that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-8mx5-7m5m-r77g" +ignoreUntil = 2026-11-20 +reason = "filelock WindowsFileLock ignores mode; the proxy image is Linux, so only Windows installs run it; there litellm's CLI takes FileLock only inside the owner-only ~/.litellm, whose lock file holds no data, and huggingface_hub takes it in the Hugging Face cache, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-9c4x-gm8v-wgc4" +ignoreUntil = 2026-11-20 +reason = "filelock StrictSoftFileLock and SoftReadWriteLock adopt another user's protocol directory, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-9j6q-wvj5-jgjg" +ignoreUntil = 2026-11-20 +reason = "filelock SoftFileLock symlink swap leaks whether a path exists; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-cwqr-hh33-gh44" +ignoreUntil = 2026-11-20 +reason = "filelock ReadWriteLock stalls on a crafted lock database, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-fq56-3cx3-fjxp" +ignoreUntil = 2026-11-20 +reason = "filelock SoftFileLock stale-lock break removes a live lock on a reused inode; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-j8f7-rjxc-mr56" +ignoreUntil = 2026-11-20 +reason = "filelock ReadWriteLock follows a symlink into SQLite, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed in 4.0.9, which litellm's own filelock<4.0 cap blocks on every Python until it is lifted (#48635, #48802); remove once that lands" + +[[IgnoredVulns]] +id = "GHSA-m462-7rxw-wvpg" +ignoreUntil = 2026-11-20 +reason = "filelock StrictSoftFileLock follows a symlink swapped in at .filelock, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-rfpc-qcw5-3pc4" +ignoreUntil = 2026-11-20 +reason = "filelock SoftFileLock removes a live holder's marker when another user forges its PID; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-pcg2-g6cj-q92c" +ignoreUntil = 2026-11-27 +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-pw9p-v7r6-gx4p" +ignoreUntil = 2026-11-27 +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-r4j5-9xpp-3985" +ignoreUntil = 2026-11-27 +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-w8v5-vhqr-4h9v" +ignoreUntil = 2026-11-27 +reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists" + +[[IgnoredVulns]] +id = "GHSA-45c3-73f7-pv4m" +ignoreUntil = 2026-11-27 +reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" + +[[IgnoredVulns]] +id = "GHSA-xcx6-jx8p-x3h5" +ignoreUntil = 2026-11-24 +reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0" [[PackageOverrides]] name = "pillow" version = "12.3.0" ecosystem = "PyPI" vulnerability.ignore = true -effectiveUntil = 2026-10-27 -reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups stays on 12.3.0; 3.11+ is on 13.0.0" +effectiveUntil = 2026-11-27 +reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups and of the mlflow extra stays on 12.3.0; 3.11+ is on 13.0.0; pillow has never shipped an older-major release after the next major (no 10.x after 11.0.0, no 11.x after 12.0.0), so 13.x is the only fix line; extended to 2026-11-27 with maintainer approval, same date as the dnspython 3.10 entry; remove once 3.10 is dropped" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e40cd134759..4188866de08 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -5326,6 +5326,58 @@ "a2a": false, "generateContent": false } + }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } } }, "endpoints": { diff --git a/pyproject.toml b/pyproject.toml index 00cdb1587aa..200bb463313 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -360,11 +360,13 @@ cache-keys = [ ] constraint-dependencies = [ "tornado>=6.5.8", + "tornado>=6.6; python_version >= '3.11'", "aiohttp>=3.14.2,<4.0", "packaging>=24.0", "soupsieve>=2.8.4", "httplib2>=0.32.0", "setuptools>=84.1.0", + "dnspython>=2.9.0; python_version >= '3.11'", ] override-dependencies = [ # a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0. diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 371884fb791..95b5942fa8b 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -18,7 +18,8 @@ # -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's # test-tree ruff and test-quality gate steps) # - dashboard -> prettier + eslint + lint budgets (test-linting.yml's ui job) -# - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (test-linting.yml's ui-api-types job) +# - spec_pattern -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift +# (test-linting.yml's ui-api-types job) # # Each block is skipped when no matching files are in scope, so unrelated commits # stay fast. This is intentionally not auto-installed as a git hook (see @@ -98,7 +99,7 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e(_harness)?/.*\.py$' test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|scripts/(check_test_quality|test_quality_gate)\.py)$' -spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' +spec_pattern='^(litellm/.*|enterprise/.*|pyproject\.toml|uv\.lock|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' @@ -110,9 +111,6 @@ e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files) -# test-linting.yml triggers on any file under litellm/proxy or litellm/types -# (Prisma schema and configs included, not just Python) plus the generator and its -# lockfiles, so match that whole trigger set rather than a Python subset. spec_files=$(scope_match "$spec_pattern") # CI's ui job runs prettier over a wider extension set than eslint; keep that # split so this flags exactly what the job would. @@ -337,7 +335,7 @@ summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_fi summary_item "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_files" \ "no tests/ Python files or test-tree lint inputs in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" -summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" +summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/, enterprise/, dependency, or generator files in scope" if [ -z "$litellm_py_files$e2e_py_files$test_tree_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2 diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 9fb3633d53c..c7781819ae4 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -107,6 +107,7 @@ ignored_function_names = [ "_deployment_num_retries", # Tested through the deployment num_retries mid-stream budget test in test_router.py "_request_fallback_list", # Tested through every mid-stream retry test in test_router.py "_request_model_group", # Tested through test_anthropic_messages_retry_budget_precedence_direct_call + "_search_tool_support", # Tested through the test_get_search_tool_support_* tests "_mid_stream_retry_trigger", # Tested through the retry policy mid-stream budget test in test_router.py "_anthropic_messages_group_retry_policy", # Tested through the retry budget precedence test in test_router.py "_anthropic_messages_resolved_retry_policy", # Tested through the malformed retry policy tests in test_router.py @@ -175,6 +176,7 @@ ignored_function_names = [ "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py + "_served_openai_models", # Cached as served_openai_models, tested through the pre-call check token count tests in test_router.py "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py "_drops_params", # Read by split_request_priority and priority_header_update, tested through both "_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py diff --git a/tests/code_coverage_tests/router_surface_allowlist.json b/tests/code_coverage_tests/router_surface_allowlist.json index 428fd9b4011..52e5bb43625 100644 --- a/tests/code_coverage_tests/router_surface_allowlist.json +++ b/tests/code_coverage_tests/router_surface_allowlist.json @@ -102,6 +102,7 @@ "get_routable_upstream_model", "get_router_deployment_budget_limiter", "get_routing_group", + "get_search_tool_support", "get_settings", "get_valid_args", "guardrail_list", diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index c3c5330c440..0e450b08ebe 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -155,3 +155,4 @@ - {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"} - {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"} - {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"} +- {id: llm.responses.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "llm_translation/test_responses_codex_tool_search_e2e.py", rationale: "Codex client tool_search over /v1/responses finds and loads a deferred tool on Claude"} diff --git a/tests/e2e/llm_translation/test_responses_codex_tool_search_e2e.py b/tests/e2e/llm_translation/test_responses_codex_tool_search_e2e.py new file mode 100644 index 00000000000..79fd61e247a --- /dev/null +++ b/tests/e2e/llm_translation/test_responses_codex_tool_search_e2e.py @@ -0,0 +1,205 @@ +"""Live e2e: Codex's client tool_search over /v1/responses on Claude. + +Codex holds MCP tools back behind its client-executed `tool_search` tool when the model catalog says +`supports_search_tool`. Over the completion bridge the proxy offers that search to Claude as a plain function, +hands Claude's call back as a `tool_search_call` Codex runs itself, and loads the tools Codex returns in +`tool_search_output` so Claude can call them on the next turn +""" + +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from openai.types.responses import ( + FunctionToolParam, + Response, + ResponseFunctionToolCall, + ResponseInputParam, + ResponseToolSearchCall, + ToolParam, + ToolSearchToolParam, +) +from proxy_client import ProxyClient +from pydantic import BaseModel, ConfigDict +from sdk_clients import NO_PROXY_CACHE, SdkClients + +pytestmark = pytest.mark.e2e + +ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-5-5" +DEFERRED_TOOL_COUNT: Final = 50 +CALENDAR_TOOL_NAME: Final = "mcp__codex_apps__calendar_create_event" + +CODEX_TOOL_SEARCH: Final[ToolSearchToolParam] = { + "type": "tool_search", + "execution": "client", + "description": ( + "# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next " + "model call. Some of the tools may not have been provided to you upfront, and you should use this tool " + "(`tool_search`) to search for the required tools." + ), + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for deferred tools."}, + "limit": {"type": "number", "description": "Maximum number of tools to return. Defaults to 8."}, + }, + "required": ["query"], + "additionalProperties": False, + }, +} + +SHELL_TOOL: Final[FunctionToolParam] = { + "type": "function", + "name": "shell", + "description": "Runs a shell command.", + "parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]}, + "strict": False, +} + +CALENDAR_TOOL: Final[FunctionToolParam] = { + "type": "function", + "name": CALENDAR_TOOL_NAME, + "description": "Create a calendar event.", + "parameters": { + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + "additionalProperties": False, + }, + "strict": False, + "defer_loading": True, +} + +PROMPT: Final = "Create a calendar event titled Standup. Find the calendar tool with tool_search first, then call it." + + +class _CalendarArguments(BaseModel): + model_config = ConfigDict(frozen=True) + + title: str + + +def _filler_tool(index: int) -> FunctionToolParam: + return { + "type": "function", + "name": f"mcp__codex_apps__records_{index:02d}", + "description": f"List, inspect, update and archive records of family {index} in the connected workspace.", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string", "description": "Title of the record to act on"}, + "notes": {"type": "string", "description": "Free-form notes and context for the operation"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "priority": {"type": "integer", "minimum": 0, "maximum": 5}, + }, + "required": ["title"], + "additionalProperties": False, + }, + "strict": False, + "defer_loading": True, + } + + +DEFERRED_TOOLS: Final[tuple[FunctionToolParam, ...]] = ( + *(_filler_tool(index) for index in range(DEFERRED_TOOL_COUNT - 1)), + CALENDAR_TOOL, +) + + +def _eager(tool: FunctionToolParam) -> ToolParam: + return FunctionToolParam( + type="function", + name=tool["name"], + description=tool.get("description"), + parameters=tool.get("parameters"), + strict=False, + ) + + +def _input_tokens(response: Response) -> int: + assert response.usage is not None, f"response carried no usage: {response.id}" + return response.usage.input_tokens + + +def _search_call(response: Response) -> ResponseToolSearchCall: + calls: Final = tuple(item for item in response.output if isinstance(item, ResponseToolSearchCall)) + assert len(calls) == 1, f"expected one tool_search_call, got {response.output!r}" + return calls[0] + + +class TestResponsesCodexToolSearchOverAnthropic: + @pytest.mark.covers("llm.responses.anthropic.tool_search.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.TOOL_SEARCH, Capability.FUNCTION_CALLING), + mode=Mode.NONSTREAM, + ) + ) + def test_claude_finds_a_deferred_tool_through_codex_tool_search_and_calls_it( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = f"e2e-codex-tool-search-{unique_marker()}" + model_id: Final = proxy.create_model( + model, LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY") + ) + resources.defer(lambda: proxy.delete_model(model_id)) + client: Final = sdk.openai(resources.key()) + searchable_tools: Final[list[ToolParam]] = [SHELL_TOOL, CODEX_TOOL_SEARCH] + + eager: Final = client.responses.create( + model=model, + input=PROMPT, + tools=[SHELL_TOOL, *(_eager(tool) for tool in DEFERRED_TOOLS)], + extra_body=NO_PROXY_CACHE, + ) + first: Final = client.responses.create( + model=model, input=PROMPT, tools=searchable_tools, extra_body=NO_PROXY_CACHE + ) + + search: Final = _search_call(first) + assert search.execution == "client", f"tool_search ran on the wrong side: {search!r}" + assert search.call_id, f"tool_search_call had no call_id: {search!r}" + assert not any(isinstance(item, ResponseFunctionToolCall) for item in first.output), first.output + assert _input_tokens(first) * 4 < _input_tokens(eager), ( + f"deferred first turn used {_input_tokens(first)} input tokens against {_input_tokens(eager)} eager" + ) + + follow_up: Final[ResponseInputParam] = [ + {"role": "user", "content": PROMPT}, + { + "type": "tool_search_call", + "call_id": search.call_id, + "execution": "client", + "arguments": search.arguments, + }, + { + "type": "tool_search_output", + "call_id": search.call_id, + "status": "completed", + "execution": "client", + "tools": [CALENDAR_TOOL], + }, + ] + second: Final = client.responses.create( + model=model, input=follow_up, tools=searchable_tools, extra_body=NO_PROXY_CACHE + ) + + calendar_call: Final = next( + ( + item + for item in second.output + if isinstance(item, ResponseFunctionToolCall) and item.name == CALENDAR_TOOL_NAME + ), + None, + ) + assert calendar_call is not None, f"Claude did not call the loaded tool: {second.output!r}" + assert "standup" in _CalendarArguments.model_validate_json(calendar_call.arguments).title.lower() diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index a727b20596c..5bdc538b2d9 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -37,3 +37,14 @@ together with its dedicated job, and list a new dedicated GitHub workflow with t `DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its directory (#46780) + +## Fixtures that boot a proxy + +Scope them to the module or class, or pin a session one to a single worker with an `xdist_group` mark +as `_llm_gateway` in `mcp/test_mcp_llm_endpoints.py` does, and request them as arguments, never through +`request.getfixturevalue`. The harness puts tests that request the same set of module or class fixtures +on one xdist worker, so the proxy they share boots there once and stops when that worker moves to another +file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot +the proxy again on another worker, so keep that set the same across the tests that share a proxy. An +unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until +the node ends, and those copies ran CircleCI integration-extensions nodes out of memory diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index b8ea11f3d2f..a759126fcd4 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -280,12 +280,14 @@ class Scenario: self.gateway.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == [] ) - def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str: - name: Final = f"integration-{uuid.uuid4().hex}" + def model( + self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue + ) -> str: + model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}" created: Final = self.gateway.post( "/model/new", { - "model_name": name, + "model_name": model_name, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", @@ -298,8 +300,8 @@ class Scenario: identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) if self.gateway.workers > 1: - self._await_every_worker(name, identity) - return name + self._await_every_worker(model_name, identity) + return model_name def _await_every_worker(self, name: str, identity: str) -> None: from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 9b990fd2167..55519678f40 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -73,6 +73,7 @@ COPILOT_USER_CODE: Final = "COPI-LEAK" COPILOT_ACCESS: Final = "device-login-github-access" COPILOT_REJECTED_ACCESS: Final = "device-login-github-access-rejected" COPILOT_FIRST_LOGIN_ACCESS: Final = "device-login-github-access-first-login" +COPILOT_UNSAVED_KEY_ACCESS: Final = "device-login-github-access-unsaved-key" COPILOT_STORED_KEY: Final = "tid=device-login;stored" COPILOT_MINTED_KEY: Final = "tid=device-login;minted" COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected" @@ -132,7 +133,11 @@ def _oauth_token(request: Request, grant: threading.Event) -> Reply: def _copilot_api_key(request: Request, api_url: str) -> Reply: - if request.headers.get("authorization") not in (f"token {COPILOT_ACCESS}", f"token {COPILOT_FIRST_LOGIN_ACCESS}"): + if request.headers.get("authorization") not in ( + f"token {COPILOT_ACCESS}", + f"token {COPILOT_FIRST_LOGIN_ACCESS}", + f"token {COPILOT_UNSAVED_KEY_ACCESS}", + ): return _json(401, {"message": "Bad credentials"}) return _json( 200, @@ -380,4 +385,7 @@ def write_copilot_access(directory: Path, access_token: str) -> None: def clear_tokens(*directories: Path) -> None: for directory in directories: for path in directory.iterdir(): - path.unlink() + if path.is_dir(): + path.rmdir() + else: + path.unlink() diff --git a/tests/integration/_support/responses_replay.py b/tests/integration/_support/responses_replay.py new file mode 100644 index 00000000000..c4e73ffbf2b --- /dev/null +++ b/tests/integration/_support/responses_replay.py @@ -0,0 +1,214 @@ +import json +import uuid +from collections.abc import Callable, Sequence +from itertools import chain +from typing import Final + +from integration._support.client import list_value, object_value +from integration._support.wire import Reply, Request, Wire +from pydantic import JsonValue + +MODEL: Final = "gpt-6.1-sol" +PLAN: Final = "I am starting by choosing which city to check." +PICK: Final = "My pick is Tokyo, so I will check its weather." +PATCH_NOTE: Final = "I will create hello.txt containing hi." +ANSWER: Final = "It is raining in Tokyo." + +WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city.", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +PATCH_TOOL: Final[dict[str, JsonValue]] = { + "type": "custom", + "custom": {"name": "apply_patch", "description": "Apply a V4A patch to the workspace"}, +} + + +def _message(item_id: str, text: str, phase: str) -> dict[str, JsonValue]: + return { + "type": "message", + "id": item_id, + "status": "completed", + "role": "assistant", + "phase": phase, + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + + +def _reasoning(item_id: str, summary: str) -> dict[str, JsonValue]: + return { + "type": "reasoning", + "id": item_id, + "summary": [{"type": "summary_text", "text": summary}], + "encrypted_content": f"enc_{item_id}", + } + + +_WEATHER_CALL: Final[dict[str, JsonValue]] = { + "type": "function_call", + "id": "fc_weather", + "call_id": "call_weather", + "name": "get_weather", + "arguments": '{"city":"Tokyo"}', + "status": "completed", +} +WEATHER_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _message("msg_plan", PLAN, "commentary"), + _reasoning("rs_pick", "Tokyo is often rainy this time of year."), + _message("msg_pick", PICK, "commentary"), + _WEATHER_CALL, +) +TWO_THOUGHTS_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _reasoning("rs_scan", "Paris and Tokyo are the two candidates."), + _reasoning("rs_pick", "Tokyo is often rainy this time of year."), + _message("msg_pick", PICK, "commentary"), + _WEATHER_CALL, +) +CALL_ONLY_TURN: Final[tuple[dict[str, JsonValue], ...]] = (_WEATHER_CALL,) +PATCH_TURN: Final[tuple[dict[str, JsonValue], ...]] = ( + _message("msg_patch", PATCH_NOTE, "commentary"), + { + "type": "custom_tool_call", + "id": "ctc_patch", + "call_id": "call_patch", + "name": "apply_patch", + "input": "*** Begin Patch\n*** Add File: hello.txt\n+hi\n*** End Patch", + "status": "completed", + }, +) +WEATHER_REPLAY: Final = ( + ("message", "assistant", "commentary", PLAN), + ("reasoning", "rs_pick"), + ("message", "assistant", "commentary", PICK), + ("function_call", "call_weather", "get_weather"), + ("function_call_output", "call_weather"), +) +TWO_THOUGHTS_REPLAY: Final = ( + ("reasoning", "rs_scan"), + ("reasoning", "rs_pick"), + ("message", "assistant", "commentary", PICK), + ("function_call", "call_weather", "get_weather"), + ("function_call_output", "call_weather"), +) + + +def _response(identity: str, output: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + return { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": MODEL, + "output": list(output), + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _event(payload: dict[str, JsonValue]) -> bytes: + return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _opened(item: dict[str, JsonValue]) -> dict[str, JsonValue]: + match item["type"]: + case "message": + return {**item, "status": "in_progress", "content": []} + case "function_call": + return {**item, "status": "in_progress", "arguments": ""} + case _: + return item + + +def _deltas(index: int, item: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + match item["type"]: + case "message": + text: Final = object_value(list_value(item["content"])[0])["text"] + return ( + { + "type": "response.output_text.delta", + "item_id": item["id"], + "output_index": index, + "content_index": 0, + "delta": text, + }, + ) + case "function_call": + return ( + { + "type": "response.function_call_arguments.delta", + "item_id": item["id"], + "output_index": index, + "delta": item["arguments"], + }, + ) + case _: + return () + + +def _item_events(index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]: + return ( + _event({"type": "response.output_item.added", "output_index": index, "item": _opened(item)}), + *(_event(delta) for delta in _deltas(index, item)), + _event({"type": "response.output_item.done", "output_index": index, "item": item}), + ) + + +def _stream(identity: str, output: Sequence[dict[str, JsonValue]]) -> tuple[bytes, ...]: + response: Final = _response(identity, output) + return ( + _event({"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}), + *chain.from_iterable(_item_events(index, item) for index, item in enumerate(output)), + _event({"type": "response.completed", "response": response}), + ) + + +def _answered(body: dict[str, JsonValue]) -> bool: + return any( + object_value(item).get("type") in {"function_call_output", "custom_tool_call_output"} + for item in list_value(body["input"]) + ) + + +def responses_upstream(turn_one: Sequence[dict[str, JsonValue]]) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + assert request.target == "/responses", request.target + body: Final = object_value(json.loads(request.body)) + identity: Final = uuid.uuid4().hex + output: Final = (_message("msg_answer", ANSWER, "final_answer"),) if _answered(body) else turn_one + if body.get("stream"): + return Reply(content_type="text/event-stream", chunks=_stream(identity, output)) + return Reply(body=json.dumps(_response(identity, output)).encode()) + + return respond + + +def _text(content: JsonValue) -> JsonValue: + if isinstance(content, str): + return content + return "".join(str(object_value(part).get("text")) for part in list_value(content)) + + +def _shape(item: JsonValue) -> tuple[JsonValue, ...]: + fields: Final = object_value(item) + match fields.get("type"): + case "message": + return ("message", fields.get("role"), fields.get("phase"), _text(fields.get("content"))) + case "reasoning": + return ("reasoning", fields.get("id")) + case "function_call" | "custom_tool_call" as kind: + return (kind, fields.get("call_id"), fields.get("name")) + case kind: + return (kind, fields.get("call_id")) + + +def posted_inputs(wire: Wire) -> tuple[tuple[tuple[JsonValue, ...], ...], ...]: + return tuple( + tuple(_shape(item) for item in list_value(object_value(json.loads(request.body))["input"])) + for request in wire.drain() + if request.method == "POST" + ) diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index a80eeacad46..4dedbdcd8e4 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -545,3 +545,94 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be assert set(token_form) == {"code", "code_verifier", "grant_type", "redirect_uri"}, token_form assert userinfo_request.method == "GET" and urlsplit(userinfo_request.target).path == "/userinfo" assert user_id == _USER_ID + + +_TEAM_OPTION: Final = re.compile(r'